
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import tree
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.experimental import enable_iterative_imputer
from sklearn.ensemble import AdaBoostClassifier, AdaBoostRegressor, GradientBoostingClassifier, GradientBoostingRegressor, RandomForestClassifier, RandomForestRegressor
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn.metrics import accuracy_score, mean_squared_error, confusion_matrix
from sklearn.preprocessing import StandardScaler
from sklearn.impute import KNNImputer, SimpleImputer, IterativeImputer


land = pd.read_csv("data/CountyData/5296US_landarea.csv")
display(land.info())
display(land.isna().sum())
display(land)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 3111 entries, 0 to 3110
Data columns (total 4 columns):
 #   Column     Non-Null Count  Dtype  
---  ------     --------------  -----  
 0   FIPS       3111 non-null   int64  
 1   State      3111 non-null   object 
 2   County     3111 non-null   object 
 3   Land Area  3111 non-null   float64
dtypes: float64(1), int64(1), object(2)
memory usage: 97.3+ KB

None

FIPS         0
State        0
County       0
Land Area    0
dtype: int64


water = pd.read_csv("data/CountyData/5296US_waterarea.csv")
display(water.info())
display(water.isna().sum())
display(water)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 3111 entries, 0 to 3110
Data columns (total 4 columns):
 #   Column      Non-Null Count  Dtype  
---  ------      --------------  -----  
 0   FIPS        3111 non-null   int64  
 1   State       3111 non-null   object 
 2   County      3111 non-null   object 
 3   Water Area  2758 non-null   float64
dtypes: float64(1), int64(1), object(2)
memory usage: 97.3+ KB

None

FIPS            0
State           0
County          0
Water Area    353
dtype: int64


pop = pd.read_csv("data/CountyData/5296US_pop.csv")
display(pop.info())
display(pop.isna().sum())
display(pop)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 3111 entries, 0 to 3110
Data columns (total 5 columns):
 #   Column      Non-Null Count  Dtype 
---  ------      --------------  ----- 
 0   FIPS        3111 non-null   int64 
 1   State       3111 non-null   object
 2   County      3111 non-null   object
 3   Date        3111 non-null   int64 
 4   Population  3111 non-null   int64 
dtypes: int64(3), object(2)
memory usage: 121.6+ KB

None

FIPS          0
State         0
County        0
Date          0
Population    0
dtype: int64


cwater = water.copy()
for col in cwater.columns:
    if cwater[col].dtype == 'object':
        cwater[col] = pd.factorize(cwater[col])[0]
knn = KNNImputer(n_neighbors=10).fit(cwater)
cwater = pd.DataFrame(knn.transform(cwater), columns=water.columns)
display(cwater.isna().sum())

FIPS          0
State         0
County        0
Water Area    0
dtype: int64


water['Water Area'] = cwater['Water Area']


countyData = pd.merge(land, pd.merge(water,pop))

# lower case headers
countyData.columns = countyData.columns.str.lower()

# lower case string values
countyData = countyData.applymap(lambda s: s.lower() if type(s) == str else s)

# check the final output
display(countyData.head(10))
display(countyData.isna().sum())

fips          0
state         0
county        0
land area     0
water area    0
date          0
population    0
dtype: int64


plt.figure(figsize=(12,12))
sns.heatmap(countyData.corr(), annot=True)
plt.title("CountyData features correlation matrix", fontsize=15, color='red')
plt.xticks(rotation = 90)
plt.show()


plt.figure(figsize=(20,20))
sns.pairplot(countyData, corner=True)
plt.show()

<Figure size 1440x1440 with 0 Axes>


plt.figure(figsize=(15,15))
sns.countplot(data=countyData, x='state')
plt.xticks(rotation = 75)
plt.show()


bcb = pd.read_csv("data/EconomicData/BCB-UDJIAD1.csv")
display(bcb.info())
display(bcb.isna().sum())
display(bcb)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 32601 entries, 0 to 32600
Data columns (total 2 columns):
 #   Column  Non-Null Count  Dtype  
---  ------  --------------  -----  
 0   Date    32601 non-null  object 
 1   Value   32601 non-null  float64
dtypes: float64(1), object(1)
memory usage: 509.5+ KB

None

Date     0
Value    0
dtype: int64


# lower case headers
bcb.columns = bcb.columns.str.lower()

bcb['date'] = pd.to_datetime(bcb['date'])
bcb['day'] = pd.to_numeric(bcb.date.dt.strftime('%d'))
bcb['month'] = pd.to_numeric(bcb.date.dt.strftime('%m'))
bcb['year'] = pd.to_numeric(bcb.date.dt.strftime('%Y'))
bcb = bcb[['day', 'month', 'year', 'value']]
display(bcb)


plt.figure(figsize=(10,10))
sns.heatmap(bcb.corr(), annot=True)
plt.title("BCB features correlation matrix", fontsize=15, color='red')
plt.xticks(rotation = 90)
plt.show()


plt.figure(figsize=(12,12))
sns.scatterplot(data=bcb, x='year', y='value', hue='month')
plt.title("Year Vs. Value, by Month", fontsize=15, color='red')
plt.xticks(rotation = 75)
plt.show()


county1 = pd.read_csv("data/EconomicData/county1.csv")
display(county1.info())
display(county1.isna().sum())
display(county1)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 3272 entries, 0 to 3271
Data columns (total 6 columns):
 #   Column                                                                     Non-Null Count  Dtype 
---  ------                                                                     --------------  ----- 
 0   Table C1.  Median Household Income by County:  1969, 1979, 1989, and 1999  3209 non-null   object
 1   Unnamed: 1                                                                 3231 non-null   object
 2   Unnamed: 2                                                                 3193 non-null   object
 3   Unnamed: 3                                                                 3193 non-null   object
 4   Unnamed: 4                                                                 3186 non-null   object
 5   Unnamed: 5                                                                 1 non-null      object
dtypes: object(6)
memory usage: 153.5+ KB

None

Table C1.  Median Household Income by County:  1969, 1979, 1989, and 1999      63
Unnamed: 1                                                                     41
Unnamed: 2                                                                     79
Unnamed: 3                                                                     79
Unnamed: 4                                                                     86
Unnamed: 5                                                                   3271
dtype: int64


county1 = county1.loc[5:3258]
county1.drop(columns=['Unnamed: 5'], inplace=True)
county1.columns = ['median household by county', 'mhs1999', 'mhs1989', 'mhs1979', 'mhs1969']
display(county1.info())
display(county1.isna().sum())
display(county1)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 3254 entries, 5 to 3258
Data columns (total 5 columns):
 #   Column                      Non-Null Count  Dtype 
---  ------                      --------------  ----- 
 0   median household by county  3200 non-null   object
 1   mhs1999                     3228 non-null   object
 2   mhs1989                     3191 non-null   object
 3   mhs1979                     3191 non-null   object
 4   mhs1969                     3184 non-null   object
dtypes: object(5)
memory usage: 127.2+ KB

None

median household by county    54
mhs1999                       26
mhs1989                       63
mhs1979                       63
mhs1969                       70
dtype: int64


for col in county1.columns:
    print(col)
    print(county1[col].unique())
    print('---------------------')

median household by county
['  United States' nan '     Alabama' ... 'Uinta County, WY'
 'Washakie County, WY' 'Weston County, WY']
---------------------
mhs1999
['41,994' nan '34,135' ... '44,544' '34,943' '32,348']
---------------------
mhs1989
['30,056' nan '23,597' ... '33,259' '25,172' '26,213']
---------------------
mhs1979
['16,841' nan '13,669' ... '22,584' '18,239' '20,021']
---------------------
mhs1969
['8,486' nan '6,419' ... '8,410' '7,588' '7,796']
---------------------


county1.replace(' ',0, inplace=True)
county1.replace(np.nan,0, inplace=True)

# check if we still have missing values
display(county1.isna().sum())

median household by county    0
mhs1999                       0
mhs1989                       0
mhs1979                       0
mhs1969                       0
dtype: int64


for col in county1.columns:
    if col == 'median household by county':
        continue
    county1[col].replace(np.nan,0,inplace=True)
    county1[col].replace(' ',0,inplace=True)
    county1[col].replace('(NA)',0,inplace=True)
    county1[col].replace('(a)',0,inplace=True)
    county1[col] = county1[col].replace({',':''},regex=True).apply(pd.to_numeric,1)
display(county1.info())

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 3254 entries, 5 to 3258
Data columns (total 5 columns):
 #   Column                      Non-Null Count  Dtype 
---  ------                      --------------  ----- 
 0   median household by county  3254 non-null   object
 1   mhs1999                     3254 non-null   int64 
 2   mhs1989                     3254 non-null   int64 
 3   mhs1979                     3254 non-null   int64 
 4   mhs1969                     3254 non-null   int64 
dtypes: int64(4), object(1)
memory usage: 127.2+ KB

None


display(county1)


county1['median_code'] = pd.factorize(county1['median household by county'])[0]
display(county1)


plt.figure(figsize=(12,12))
sns.heatmap(county1.corr(), annot=True)
plt.title("County1 features correlation matrix", fontsize=15, color='red')
plt.xticks(rotation = 90)
plt.show()


plt.figure(figsize=(20,20))
sns.pairplot(county1, corner=True)
plt.show()

<Figure size 1440x1440 with 0 Axes>


dow = pd.read_csv("data/EconomicData/Dow Jones Industrial Average Historical Data 2.csv")
display(dow.info())
display(dow.isna().sum())
display(dow)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 2766 entries, 0 to 2765
Data columns (total 7 columns):
 #   Column    Non-Null Count  Dtype 
---  ------    --------------  ----- 
 0   Date      2766 non-null   object
 1   Price     2766 non-null   object
 2   Open      2766 non-null   object
 3   High      2766 non-null   object
 4   Low       2766 non-null   object
 5   Vol.      2766 non-null   object
 6   Change %  2766 non-null   object
dtypes: object(7)
memory usage: 151.4+ KB

None

Date        0
Price       0
Open        0
High        0
Low         0
Vol.        0
Change %    0
dtype: int64


# lower case headers
dow.columns = dow.columns.str.lower()

dow['date'] = pd.to_datetime(dow['date'])
dow['day'] = pd.to_numeric(dow.date.dt.strftime('%d'))
dow['month'] = pd.to_numeric(dow.date.dt.strftime('%m'))
dow['year'] = pd.to_numeric(dow.date.dt.strftime('%Y'))
dow = dow[['day', 'month', 'year', 'price', 'open', 'high', 'low', 'vol.', 'change %']]
display(dow.info())
display(dow)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 2766 entries, 0 to 2765
Data columns (total 9 columns):
 #   Column    Non-Null Count  Dtype 
---  ------    --------------  ----- 
 0   day       2766 non-null   int64 
 1   month     2766 non-null   int64 
 2   year      2766 non-null   int64 
 3   price     2766 non-null   object
 4   open      2766 non-null   object
 5   high      2766 non-null   object
 6   low       2766 non-null   object
 7   vol.      2766 non-null   object
 8   change %  2766 non-null   object
dtypes: int64(3), object(6)
memory usage: 194.6+ KB

None


# rename vol. to vol. (millions) since we are going to drop M
dow.rename(columns={'vol.': 'vol. (millions)'}, inplace=True)
display(dow)

C:\Users\lover\anaconda3\lib\site-packages\pandas\core\frame.py:5047: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  errors=errors,


dow = dow.replace({',':'', 'M': '', '%': '', '-':0},regex=True).apply(pd.to_numeric,1)
display(dow.info())
display(dow.isna().sum())
display(dow)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 2766 entries, 0 to 2765
Data columns (total 9 columns):
 #   Column           Non-Null Count  Dtype  
---  ------           --------------  -----  
 0   day              2766 non-null   float64
 1   month            2766 non-null   float64
 2   year             2766 non-null   float64
 3   price            2766 non-null   float64
 4   open             2766 non-null   float64
 5   high             2766 non-null   float64
 6   low              2766 non-null   float64
 7   vol. (millions)  2766 non-null   float64
 8   change %         2766 non-null   float64
dtypes: float64(9)
memory usage: 194.6 KB

None

day                0
month              0
year               0
price              0
open               0
high               0
low                0
vol. (millions)    0
change %           0
dtype: int64


plt.figure(figsize=(15,15))
sns.heatmap(dow.corr(), annot=True)
plt.title("Dow Jones features correlation matrix", fontsize=15, color='red')
plt.xticks(rotation =90)
plt.show()


dow.drop(columns=['year', 'open', 'high', 'low'], inplace=True)
plt.figure(figsize=(20,20))
sns.pairplot(dow, corner=True)
plt.show()

<Figure size 1440x1440 with 0 Axes>


gdp = pd.read_excel("data/EconomicData/gdpcounty1218.xlsx")
display(gdp.info())
display(gdp.isna().sum())
display(gdp)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 12459 entries, 0 to 12458
Data columns (total 9 columns):
 #   Column      Non-Null Count  Dtype 
---  ------      --------------  ----- 
 0   Unnamed: 0  12456 non-null  object
 1   Unnamed: 1  12453 non-null  object
 2   Unnamed: 2  12453 non-null  object
 3   Unnamed: 3  12453 non-null  object
 4   Unnamed: 4  12453 non-null  object
 5   Unnamed: 5  12455 non-null  object
 6   Unnamed: 6  12453 non-null  object
 7   Unnamed: 7  12453 non-null  object
 8   Unnamed: 8  12453 non-null  object
dtypes: object(9)
memory usage: 876.1+ KB

None

Unnamed: 0    3
Unnamed: 1    6
Unnamed: 2    6
Unnamed: 3    6
Unnamed: 4    6
Unnamed: 5    4
Unnamed: 6    6
Unnamed: 7    6
Unnamed: 8    6
dtype: int64


gdp = gdp.loc[3:12454]
gdp.columns = ['fips', 'county', 'postal', 'line_code', 'industry_name', 'gdp2012', 'gdp2013', 'gdp2014', 'gdp2015']
display(gdp.info())
display(gdp.isna().sum())
display(gdp)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 12452 entries, 3 to 12454
Data columns (total 9 columns):
 #   Column         Non-Null Count  Dtype 
---  ------         --------------  ----- 
 0   fips           12452 non-null  object
 1   county         12452 non-null  object
 2   postal         12452 non-null  object
 3   line_code      12452 non-null  object
 4   industry_name  12452 non-null  object
 5   gdp2012        12452 non-null  object
 6   gdp2013        12452 non-null  object
 7   gdp2014        12452 non-null  object
 8   gdp2015        12452 non-null  object
dtypes: object(9)
memory usage: 875.7+ KB

None

fips             0
county           0
postal           0
line_code        0
industry_name    0
gdp2012          0
gdp2013          0
gdp2014          0
gdp2015          0
dtype: int64


gdps = gdp[['gdp2012', 'gdp2013', 'gdp2014', 'gdp2015']]
display(gdps)


gdps = gdps.replace({np.nan:0, ' ':0, '-':0, '(D)':0},regex=True).apply(pd.to_numeric,1)
display(gdps.info())

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 12452 entries, 3 to 12454
Data columns (total 4 columns):
 #   Column   Non-Null Count  Dtype
---  ------   --------------  -----
 0   gdp2012  12452 non-null  int64
 1   gdp2013  12452 non-null  int64
 2   gdp2014  12452 non-null  int64
 3   gdp2015  12452 non-null  int64
dtypes: int64(4)
memory usage: 389.3 KB

None


gdp['fips'].unique()

array(['01001', '01003', '01005', ..., '56041', '56043', '56045'],
      dtype=object)


gdp['fips'] = pd.to_numeric(gdp['fips'])


gdp['line_code'] = pd.to_numeric(gdp['line_code'])


gdp = pd.concat([gdp[['fips','county','postal', 'line_code', 'industry_name']], gdps], axis = 1)
display(gdp.info())
display(gdp.isna().sum())
display(gdp)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 12452 entries, 3 to 12454
Data columns (total 9 columns):
 #   Column         Non-Null Count  Dtype 
---  ------         --------------  ----- 
 0   fips           12452 non-null  int64 
 1   county         12452 non-null  object
 2   postal         12452 non-null  object
 3   line_code      12452 non-null  int64 
 4   industry_name  12452 non-null  object
 5   gdp2012        12452 non-null  int64 
 6   gdp2013        12452 non-null  int64 
 7   gdp2014        12452 non-null  int64 
 8   gdp2015        12452 non-null  int64 
dtypes: int64(6), object(3)
memory usage: 875.7+ KB

None

fips             0
county           0
postal           0
line_code        0
industry_name    0
gdp2012          0
gdp2013          0
gdp2014          0
gdp2015          0
dtype: int64


plt.figure(figsize=(15,15))
sns.heatmap(gdp.corr(), annot=True)
plt.title("Gross domestic product (GDP) by county features correlation matrix", fontsize=15, color='red')
plt.xticks(rotation = 90)
plt.show()


gdp.drop(columns=['gdp2012', 'gdp2013', 'gdp2014'], inplace=True)
plt.figure(figsize=(20,20))
sns.pairplot(gdp, corner=True)
plt.show()

<Figure size 1440x1440 with 0 Axes>


plt.figure(figsize=(15,15))
sns.scatterplot(data=gdp, x='county', y='gdp2015', hue='industry_name')
plt.title("county Vs. gdp2015, by industry_name", fontsize=15, color='red')
plt.xticks(rotation = 75)
plt.show()


complete = pd.read_csv("data/GeneralDemographicData/county_complete.csv")
display(complete.info())
display(complete.isna().sum())
display(complete)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 3142 entries, 0 to 3141
Columns: 188 entries, fips to white_not_hispanic_2019
dtypes: float64(166), int64(18), object(4)
memory usage: 4.5+ MB

None

fips                       0
state                      0
name                       0
pop2000                    3
pop2010                    0
                          ..
uninsured_under_19_2019    0
uninsured_under_6_2019     0
veterans_2019              0
white_2019                 0
white_not_hispanic_2019    0
Length: 188, dtype: int64


complete.replace({'-':''},regex=True, inplace=True)
complete['uninsured_age_under_6_2017'] = pd.to_numeric(complete['uninsured_age_under_6_2017'])
cpObject = complete.select_dtypes('object')
cpNumeric = complete.select_dtypes(exclude='object')
display(cpObject)
display(cpNumeric)


knn = KNNImputer(n_neighbors=30).fit(cpNumeric)
imputed = pd.DataFrame(knn.transform(cpNumeric), columns = cpNumeric.columns)
display(imputed)


complete = pd.concat([cpObject, imputed], axis=1)
display(complete.info())
display(complete.isna().sum())
display(complete)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 3142 entries, 0 to 3141
Columns: 188 entries, state to white_not_hispanic_2019
dtypes: float64(185), object(3)
memory usage: 4.5+ MB

None

state                       0
name                        0
smoking_ban_2010           26
fips                        0
pop2000                     0
                           ..
uninsured_under_19_2019     0
uninsured_under_6_2019      0
veterans_2019               0
white_2019                  0
white_not_hispanic_2019     0
Length: 188, dtype: int64


complete['smoking_ban_2010'].unique()

array(['none', 'partial', 'comprehensive', nan], dtype=object)


complete.replace(np.nan, 'none', inplace=True)
display(complete.isna().sum())

state                      0
name                       0
smoking_ban_2010           0
fips                       0
pop2000                    0
                          ..
uninsured_under_19_2019    0
uninsured_under_6_2019     0
veterans_2019              0
white_2019                 0
white_not_hispanic_2019    0
Length: 188, dtype: int64


# lower case string values
complete = complete.applymap(lambda s: s.lower() if type(s)==str else s)
display(complete.sample(20))


plt.figure(figsize=(15,15))
sns.scatterplot(data=complete, x='state', y='uninsured_age_under_6_2017', hue='smoking_ban_2010')
plt.title("State Vs. uninsured_age_under_6_2017, by smoking_ban_2010", fontsize=15, color='red')
plt.xticks(rotation = 75)
plt.show()


life = pd.read_excel("data/LifeExpectancyData/IHME_USA_LIFE_EXPECTANCY_1987_2007_Y2011M06D16.XLSX")
display(life.info())
display(life.isna().sum())
display(life)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 66087 entries, 0 to 66086
Data columns (total 24 columns):
 #   Column                                                        Non-Null Count  Dtype  
---  ------                                                        --------------  -----  
 0   fips                                                          66087 non-null  int64  
 1   State                                                         66087 non-null  object 
 2   County                                                        66087 non-null  object 
 3   Year                                                          66087 non-null  int64  
 4   Male life expectancy (years)                                  66087 non-null  float64
 5   Years behind international frontier (male)                    66087 non-null  object 
 6   Female life expectancy (years)                                66087 non-null  float64
 7   Years behind international frontier (female)                  66087 non-null  object 
 8   White male life expectancy (years)                            66087 non-null  float64
 9   White female life expectancy (years)                          66087 non-null  float64
 10  Black male life expectancy (years)                            15494 non-null  float64
 11  Black female life expectancy (years)                          18362 non-null  float64
 12  Closest ranked countries for male life expectancy (higher)    65991 non-null  object 
 13  Closest ranked countries for female life expectancy (higher)  66076 non-null  object 
 14  Closest ranked countries for male life expectancy (lower)     66087 non-null  object 
 15  Closest ranked countries for female life expectancy (lower)   66087 non-null  object 
 16  Rank (male)                                                   66087 non-null  int64  
 17  Rank (female)                                                 66087 non-null  int64  
 18  Male life expectancy change 1987 to 2007 (years)              66087 non-null  float64
 19  Female life expectancy change 1987 to 2007 (years)            66087 non-null  float64
 20  Male life expectancy change 1987 to 1997 (years)              66087 non-null  float64
 21  Female life expectancy change 1987 to 1997 (years)            66087 non-null  float64
 22  Male life expectancy change 1997 to 2007 (years)              66087 non-null  float64
 23  Female life expectancy change 1997 to 2007 (years)            66087 non-null  float64
dtypes: float64(12), int64(4), object(8)
memory usage: 12.1+ MB

None

fips                                                                0
State                                                               0
County                                                              0
Year                                                                0
Male life expectancy (years)                                        0
Years behind international frontier (male)                          0
Female life expectancy (years)                                      0
Years behind international frontier (female)                        0
White male life expectancy (years)                                  0
White female life expectancy (years)                                0
Black male life expectancy (years)                              50593
Black female life expectancy (years)                            47725
Closest ranked countries for male life expectancy (higher)         96
Closest ranked countries for female life expectancy (higher)       11
Closest ranked countries for male life expectancy (lower)           0
Closest ranked countries for female life expectancy (lower)         0
Rank (male)                                                         0
Rank (female)                                                       0
Male life expectancy change 1987 to 2007 (years)                    0
Female life expectancy change 1987 to 2007 (years)                  0
Male life expectancy change 1987 to 1997 (years)                    0
Female life expectancy change 1987 to 1997 (years)                  0
Male life expectancy change 1997 to 2007 (years)                    0
Female life expectancy change 1997 to 2007 (years)                  0
dtype: int64


lifemissings = life[['Black male life expectancy (years)', 'Black female life expectancy (years)', 'Closest ranked countries for male life expectancy (higher)', 'Closest ranked countries for female life expectancy (higher)']]
display(lifemissings)


for col in lifemissings.columns:
    if col == 'Closest ranked countries for male life expectancy (higher)' or col == 'Closest ranked countries for femmale life expectancy (higher)':
        lifemissings[col].replace(np.nan, 'none', inplace = True)
    else :
        lifemissings[col].replace(np.nan,0,inplace=True)
display(lifemissings.info())
display(lifemissings)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 66087 entries, 0 to 66086
Data columns (total 4 columns):
 #   Column                                                        Non-Null Count  Dtype  
---  ------                                                        --------------  -----  
 0   Black male life expectancy (years)                            66087 non-null  float64
 1   Black female life expectancy (years)                          66087 non-null  float64
 2   Closest ranked countries for male life expectancy (higher)    66087 non-null  object 
 3   Closest ranked countries for female life expectancy (higher)  66087 non-null  object 
dtypes: float64(2), object(2)
memory usage: 2.0+ MB

None


lifemissings.isna().sum()

Black male life expectancy (years)                              0
Black female life expectancy (years)                            0
Closest ranked countries for male life expectancy (higher)      0
Closest ranked countries for female life expectancy (higher)    0
dtype: int64


for col in life.columns:
    if col in (lifemissings.columns):
        life[col] = lifemissings[col]
display(life.info())
display(life.isna().sum())
display(life.sample(20))

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 66087 entries, 0 to 66086
Data columns (total 24 columns):
 #   Column                                                        Non-Null Count  Dtype  
---  ------                                                        --------------  -----  
 0   fips                                                          66087 non-null  int64  
 1   State                                                         66087 non-null  object 
 2   County                                                        66087 non-null  object 
 3   Year                                                          66087 non-null  int64  
 4   Male life expectancy (years)                                  66087 non-null  float64
 5   Years behind international frontier (male)                    66087 non-null  object 
 6   Female life expectancy (years)                                66087 non-null  float64
 7   Years behind international frontier (female)                  66087 non-null  object 
 8   White male life expectancy (years)                            66087 non-null  float64
 9   White female life expectancy (years)                          66087 non-null  float64
 10  Black male life expectancy (years)                            66087 non-null  float64
 11  Black female life expectancy (years)                          66087 non-null  float64
 12  Closest ranked countries for male life expectancy (higher)    66087 non-null  object 
 13  Closest ranked countries for female life expectancy (higher)  66087 non-null  object 
 14  Closest ranked countries for male life expectancy (lower)     66087 non-null  object 
 15  Closest ranked countries for female life expectancy (lower)   66087 non-null  object 
 16  Rank (male)                                                   66087 non-null  int64  
 17  Rank (female)                                                 66087 non-null  int64  
 18  Male life expectancy change 1987 to 2007 (years)              66087 non-null  float64
 19  Female life expectancy change 1987 to 2007 (years)            66087 non-null  float64
 20  Male life expectancy change 1987 to 1997 (years)              66087 non-null  float64
 21  Female life expectancy change 1987 to 1997 (years)            66087 non-null  float64
 22  Male life expectancy change 1997 to 2007 (years)              66087 non-null  float64
 23  Female life expectancy change 1997 to 2007 (years)            66087 non-null  float64
dtypes: float64(12), int64(4), object(8)
memory usage: 12.1+ MB

None

fips                                                            0
State                                                           0
County                                                          0
Year                                                            0
Male life expectancy (years)                                    0
Years behind international frontier (male)                      0
Female life expectancy (years)                                  0
Years behind international frontier (female)                    0
White male life expectancy (years)                              0
White female life expectancy (years)                            0
Black male life expectancy (years)                              0
Black female life expectancy (years)                            0
Closest ranked countries for male life expectancy (higher)      0
Closest ranked countries for female life expectancy (higher)    0
Closest ranked countries for male life expectancy (lower)       0
Closest ranked countries for female life expectancy (lower)     0
Rank (male)                                                     0
Rank (female)                                                   0
Male life expectancy change 1987 to 2007 (years)                0
Female life expectancy change 1987 to 2007 (years)              0
Male life expectancy change 1987 to 1997 (years)                0
Female life expectancy change 1987 to 1997 (years)              0
Male life expectancy change 1997 to 2007 (years)                0
Female life expectancy change 1997 to 2007 (years)              0
dtype: int64


life.columns = life.columns.str.lower()
life = life.applymap(lambda s : s.lower() if type(s)==str else s)
life


aggregated = life.groupby(['county', 'year']).mean()
aggregated


house = pd.read_csv("data/USElectionResults19762020/1976-2020-house.csv")
display(house.info())
display(house.isna().sum())
display(house)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 31103 entries, 0 to 31102
Data columns (total 20 columns):
 #   Column          Non-Null Count  Dtype 
---  ------          --------------  ----- 
 0   year            31103 non-null  int64 
 1   state           31103 non-null  object
 2   state_po        31103 non-null  object
 3   state_fips      31103 non-null  int64 
 4   state_cen       31103 non-null  int64 
 5   state_ic        31103 non-null  int64 
 6   office          31103 non-null  object
 7   district        31103 non-null  int64 
 8   stage           31103 non-null  object
 9   runoff          22447 non-null  object
 10  special         31103 non-null  bool  
 11  candidate       31103 non-null  object
 12  party           27483 non-null  object
 13  writein         31103 non-null  bool  
 14  mode            31103 non-null  object
 15  candidatevotes  31103 non-null  int64 
 16  totalvotes      31103 non-null  int64 
 17  unofficial      31103 non-null  bool  
 18  version         31103 non-null  int64 
 19  fusion_ticket   31103 non-null  bool  
dtypes: bool(4), int64(8), object(8)
memory usage: 3.9+ MB

None

year                 0
state                0
state_po             0
state_fips           0
state_cen            0
state_ic             0
office               0
district             0
stage                0
runoff            8656
special              0
candidate            0
party             3620
writein              0
mode                 0
candidatevotes       0
totalvotes           0
unofficial           0
version              0
fusion_ticket        0
dtype: int64


house.replace({np.nan:'unknown', '':'unknown'}, regex=True,inplace=True)
house.isna().sum()

year              0
state             0
state_po          0
state_fips        0
state_cen         0
state_ic          0
office            0
district          0
stage             0
runoff            0
special           0
candidate         0
party             0
writein           0
mode              0
candidatevotes    0
totalvotes        0
unofficial        0
version           0
fusion_ticket     0
dtype: int64


house.columns = house.columns.str.lower()
house = house.applymap(lambda s: s.lower() if type(s)==str else s)
display(house.info())
display(house)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 31103 entries, 0 to 31102
Data columns (total 20 columns):
 #   Column          Non-Null Count  Dtype 
---  ------          --------------  ----- 
 0   year            31103 non-null  int64 
 1   state           31103 non-null  object
 2   state_po        31103 non-null  object
 3   state_fips      31103 non-null  int64 
 4   state_cen       31103 non-null  int64 
 5   state_ic        31103 non-null  int64 
 6   office          31103 non-null  object
 7   district        31103 non-null  int64 
 8   stage           31103 non-null  object
 9   runoff          31103 non-null  object
 10  special         31103 non-null  bool  
 11  candidate       31103 non-null  object
 12  party           31103 non-null  object
 13  writein         31103 non-null  bool  
 14  mode            31103 non-null  object
 15  candidatevotes  31103 non-null  int64 
 16  totalvotes      31103 non-null  int64 
 17  unofficial      31103 non-null  bool  
 18  version         31103 non-null  int64 
 19  fusion_ticket   31103 non-null  bool  
dtypes: bool(4), int64(8), object(8)
memory usage: 3.9+ MB

None


plt.figure(figsize=(20,20))
sns.heatmap(house.corr(), annot=True)
plt.title("House features correlation matrix", fontsize=15, color='red')
plt.xticks(rotation = 90)
plt.show()


plt.figure(figsize=(25,25))
sns.pairplot(house, corner=True)
plt.show()

<string>:6: RuntimeWarning: Converting input from bool to <class 'numpy.uint8'> for compatibility.
<string>:6: RuntimeWarning: Converting input from bool to <class 'numpy.uint8'> for compatibility.
<string>:6: RuntimeWarning: Converting input from bool to <class 'numpy.uint8'> for compatibility.
<string>:6: RuntimeWarning: Converting input from bool to <class 'numpy.uint8'> for compatibility.
<string>:6: RuntimeWarning: Converting input from bool to <class 'numpy.uint8'> for compatibility.
<string>:6: RuntimeWarning: Converting input from bool to <class 'numpy.uint8'> for compatibility.
<string>:6: RuntimeWarning: Converting input from bool to <class 'numpy.uint8'> for compatibility.
<string>:6: RuntimeWarning: Converting input from bool to <class 'numpy.uint8'> for compatibility.

<Figure size 1800x1800 with 0 Axes>


president = pd.read_csv("data/USElectionResults19762020/1976-2020-president.csv")
display(president.info())
display(president.isna().sum())
display(president)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 4287 entries, 0 to 4286
Data columns (total 15 columns):
 #   Column            Non-Null Count  Dtype  
---  ------            --------------  -----  
 0   year              4287 non-null   int64  
 1   state             4287 non-null   object 
 2   state_po          4287 non-null   object 
 3   state_fips        4287 non-null   int64  
 4   state_cen         4287 non-null   int64  
 5   state_ic          4287 non-null   int64  
 6   office            4287 non-null   object 
 7   candidate         4000 non-null   object 
 8   party_detailed    3831 non-null   object 
 9   writein           4284 non-null   object 
 10  candidatevotes    4287 non-null   int64  
 11  totalvotes        4287 non-null   int64  
 12  version           4287 non-null   int64  
 13  notes             0 non-null      float64
 14  party_simplified  4287 non-null   object 
dtypes: float64(1), int64(7), object(7)
memory usage: 502.5+ KB

None

year                   0
state                  0
state_po               0
state_fips             0
state_cen              0
state_ic               0
office                 0
candidate            287
party_detailed       456
writein                3
candidatevotes         0
totalvotes             0
version                0
notes               4287
party_simplified       0
dtype: int64


president.drop(columns=['notes'], inplace=True) #all nan feature
president.replace({'': 'unknown', np.nan:'unknown'},regex=True, inplace=True)
display(president.isna().sum())

year                0
state               0
state_po            0
state_fips          0
state_cen           0
state_ic            0
office              0
candidate           0
party_detailed      0
writein             0
candidatevotes      0
totalvotes          0
version             0
party_simplified    0
dtype: int64


president.columns = president.columns.str.lower()
president = president.applymap(lambda s: s.lower() if type(s)==str else s)


plt.figure(figsize=(20,20))
sns.heatmap(president.corr(), annot=True)
plt.title("President features correlation matrix", fontsize=15, color='red')
plt.xticks(rotation = 90)
plt.show()


plt.figure(figsize=(25,25))
sns.pairplot(president, corner=True)
plt.show()

<Figure size 1800x1800 with 0 Axes>


senate = pd.read_csv("data/USElectionResults19762020/1976-2020-senate.csv")
display(senate.info())
display(senate.isna().sum())
display(senate)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 3629 entries, 0 to 3628
Data columns (total 19 columns):
 #   Column            Non-Null Count  Dtype 
---  ------            --------------  ----- 
 0   year              3629 non-null   int64 
 1   state             3629 non-null   object
 2   state_po          3629 non-null   object
 3   state_fips        3629 non-null   int64 
 4   state_cen         3629 non-null   int64 
 5   state_ic          3629 non-null   int64 
 6   office            3629 non-null   object
 7   district          3629 non-null   object
 8   stage             3629 non-null   object
 9   special           3629 non-null   bool  
 10  candidate         3206 non-null   object
 11  party_detailed    3030 non-null   object
 12  writein           3629 non-null   bool  
 13  mode              3629 non-null   object
 14  candidatevotes    3629 non-null   int64 
 15  totalvotes        3629 non-null   int64 
 16  unofficial        3629 non-null   bool  
 17  version           3629 non-null   int64 
 18  party_simplified  3629 non-null   object
dtypes: bool(3), int64(7), object(9)
memory usage: 464.4+ KB

None

year                  0
state                 0
state_po              0
state_fips            0
state_cen             0
state_ic              0
office                0
district              0
stage                 0
special               0
candidate           423
party_detailed      599
writein               0
mode                  0
candidatevotes        0
totalvotes            0
unofficial            0
version               0
party_simplified      0
dtype: int64


senate.replace({np.nan:'unknown', '':'unknown'},regex=True, inplace=True)
senate.isna().sum()

year                0
state               0
state_po            0
state_fips          0
state_cen           0
state_ic            0
office              0
district            0
stage               0
special             0
candidate           0
party_detailed      0
writein             0
mode                0
candidatevotes      0
totalvotes          0
unofficial          0
version             0
party_simplified    0
dtype: int64


plt.figure(figsize=(20,20))
sns.heatmap(senate.corr(), annot=True)
plt.title("Senate features correlation matrix", fontsize=15, color='red')
plt.xticks(rotation = 90)
plt.show()


plt.figure(figsize=(25,25))
sns.pairplot(senate, corner=True)
plt.show()

<string>:6: RuntimeWarning: Converting input from bool to <class 'numpy.uint8'> for compatibility.
<string>:6: RuntimeWarning: Converting input from bool to <class 'numpy.uint8'> for compatibility.
<string>:6: RuntimeWarning: Converting input from bool to <class 'numpy.uint8'> for compatibility.
<string>:6: RuntimeWarning: Converting input from bool to <class 'numpy.uint8'> for compatibility.
<string>:6: RuntimeWarning: Converting input from bool to <class 'numpy.uint8'> for compatibility.
<string>:6: RuntimeWarning: Converting input from bool to <class 'numpy.uint8'> for compatibility.

<Figure size 1800x1800 with 0 Axes>


# merge the three dataframes we were asked to use
sectionB = pd.merge(countyData, pd.merge(complete,life))
display(sectionB.info())
display(sectionB)

<class 'pandas.core.frame.DataFrame'>
Int64Index: 61110 entries, 0 to 61109
Columns: 214 entries, fips to female life expectancy change 1997 to 2007 (years)
dtypes: float64(198), int64(6), object(10)
memory usage: 100.2+ MB

None


# We can use only : california, florida, south dakota and wyoming, so we will filter state to include only this states
sectionB = sectionB[sectionB['state'].isin(['california', 'florida', 'south dakota', 'wyoming'])]
display(sectionB.info())

<class 'pandas.core.frame.DataFrame'>
Int64Index: 4263 entries, 3276 to 61109
Columns: 214 entries, fips to female life expectancy change 1997 to 2007 (years)
dtypes: float64(198), int64(6), object(10)
memory usage: 7.0+ MB

None


sectionBI = sectionB.reset_index(drop=True)
sectionBI


sectionBI['state'].unique()

array(['california', 'florida', 'south dakota', 'wyoming'], dtype=object)


nsectionBI = sectionBI.copy()
for col in nsectionBI.columns:
    if nsectionBI[col].dtype == 'object':
        nsectionBI[col] = pd.factorize(nsectionBI[col])[0]
nsectionBI


nsectionBI = pd.DataFrame(StandardScaler().fit_transform(nsectionBI), columns=nsectionBI.columns)
nsectionBI


pca = PCA(n_components=20).fit(nsectionBI)
per_var = np.round(pca.explained_variance_ratio_ * 100, decimals= 1)
labels = ['PC' + str(x) for x in range(1,len(per_var)+1)]
npcaDF = pd.DataFrame(pca.transform(nsectionBI), columns=labels)


npcaDF


npcaDF = pd.concat([sectionBI[['state']], npcaDF], axis = 1)
npcaDF


fig = plt.figure(figsize = (8,8))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('Principal Component 1', fontsize = 15)
ax.set_ylabel('Principal Component 2', fontsize = 15)
ax.set_title('2 component PCA', fontsize = 20)
targets = ['california', 'florida', 'south dakota', 'wyoming']
colors = ['r', 'g', 'b','y']
for target, color in zip(targets,colors):
    indicesToKeep = npcaDF['state'] == target
    ax.scatter(npcaDF.loc[indicesToKeep, 'PC1']
               , npcaDF.loc[indicesToKeep, 'PC2']
               , c = color
               , s = 50)
ax.legend(targets)
ax.grid()


loading_scores = pd.Series(pca.components_[0], index= sectionBI.columns)
sorted_loading_scores = loading_scores.abs().sort_values(ascending=False)
top10Features = sorted_loading_scores[0:10].index.values
print(loading_scores[top10Features])

households_2019              0.112029
pop_2019                     0.112009
pop2017                      0.111987
households_2017              0.111973
pop2016                      0.111936
employed_2015                0.111926
civilian_labor_force_2016    0.111910
civilian_labor_force_2017    0.111897
civilian_labor_force_2015    0.111888
pop2015                      0.111882
dtype: float64


bottom10Features = sorted_loading_scores[-10:].index.values
print(loading_scores[bottom10Features])

poverty_2017                                     -5.193086e-03
years behind international frontier (male)       -4.596984e-03
percent_change_private_nonfarm_employment_2009   -4.361116e-03
age_under_5_2017                                  4.272801e-03
hispanic_owned_firms_2007                        -2.907996e-03
poverty_age_under_18_2017                        -1.610122e-03
age_under_5_2019                                  6.553234e-04
year                                              5.613648e-04
age_over_18_2019                                  3.424495e-04
date                                              2.019484e-28
dtype: float64


complete


fcomplete = complete.copy()
fcomplete = fcomplete[fcomplete['state'].isin(['california', 'florida', 'south dakota', 'wyoming'])]
fcomplete


fcomplete.reset_index(drop=True,inplace=True)
fcomplete


nfcomplete = fcomplete.copy()
for col in nfcomplete.columns:
    if nfcomplete[col].dtype == 'object':
        nfcomplete[col] = pd.factorize(nfcomplete[col])[0]
nfcomplete = pd.DataFrame(StandardScaler().fit_transform(nfcomplete), columns=nfcomplete.columns)
nfcomplete


compPCA = PCA(n_components=5).fit(nfcomplete)
per_var = np.round(compPCA.explained_variance_ratio_ * 100, decimals= 1)
labels = ['PC' + str(x) for x in range(1,len(per_var)+1)]
compDF = pd.DataFrame(compPCA.transform(nfcomplete), columns=labels)
compDF


comp_loading_scores = pd.Series(compPCA.components_[0], index= nfcomplete.columns)
sorted_comp_loading_scores = comp_loading_scores.abs().sort_values(ascending=False)
top10Features = sorted_comp_loading_scores[0:10].index.values
print(comp_loading_scores[top10Features])

households_2019              0.114686
employed_2015                0.114685
civilian_labor_force_2016    0.114678
civilian_labor_force_2017    0.114664
civilian_labor_force_2015    0.114659
pop_2019                     0.114658
pop2017                      0.114644
employed_2017                0.114639
households_2017              0.114638
employed_2016                0.114635
dtype: float64


bottom10Features = sorted_comp_loading_scores[-10:].index.values
print(comp_loading_scores[bottom10Features])

poverty_2016                                     -0.006072
age_under_5_2017                                  0.005694
households_speak_other_2019                      -0.005034
percent_change_private_nonfarm_employment_2009   -0.004929
poverty_age_under_5_2017                         -0.003414
uninsured_2017                                   -0.003142
age_under_5_2019                                  0.001684
poverty_age_under_18_2017                         0.001509
poverty_2017                                     -0.001373
age_over_18_2019                                  0.000339
dtype: float64


compDF = pd.concat([fcomplete[['state']], compDF], axis = 1)
compDF


fig = plt.figure(figsize = (8,8))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('Principal Component 1', fontsize = 15)
ax.set_ylabel('Principal Component 2', fontsize = 15)
ax.set_title('2 component PCA - County Complete', fontsize = 20)
targets = ['california', 'florida', 'south dakota', 'wyoming']
colors = ['r', 'g', 'b','y']
for target, color in zip(targets,colors):
    indicesToKeep = compDF['state'] == target
    ax.scatter(compDF.loc[indicesToKeep, 'PC1']
               , compDF.loc[indicesToKeep, 'PC2']
               , c = color
               , s = 50)
ax.legend(targets)
ax.grid()


countyData


ncd = countyData.copy()
ncd = ncd[ncd['state'].isin(['california', 'florida', 'south dakota', 'wyoming'])]
ncd.reset_index(drop=True,inplace=True)


ncd = countyData.copy()
for col in ncd.columns:
    if ncd[col].dtype == 'object':
        ncd[col] = pd.factorize(ncd[col])[0]
ncd = pd.DataFrame(StandardScaler().fit_transform(ncd), columns=ncd.columns)
ncd


cdpca = PCA(n_components=7).fit(ncd)
per_var2 = np.round(cdpca.explained_variance_ratio_ * 100, decimals= 1)
labels2 = ['PC' + str(x) for x in range(1,len(per_var2)+1)]
ncdDF = pd.DataFrame(cdpca.transform(ncd), columns = labels2)
ncdDF


states = countyData[countyData['state'].isin(['california', 'florida', 'south dakota', 'wyoming'])]
states.reset_index(drop=True,inplace=True)
ncdDF2 = pd.concat([states[['state']], ncdDF], axis = 1)
ncdDF2


ncdDF2.state.unique()

array(['california', 'florida', 'south dakota', 'wyoming', nan],
      dtype=object)


fig = plt.figure(figsize = (8,8))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('Principal Component 1', fontsize = 15)
ax.set_ylabel('Principal Component 2', fontsize = 15)
ax.set_title('2 component PCA - CountyData', fontsize = 20)
targets = ['california', 'florida', 'south dakota', 'wyoming', np.nan]
colors = ['r', 'g', 'b','y', 'black']
for target, color in zip(targets,colors):
    indicesToKeep = ncdDF2['state'] == target
    ax.scatter(ncdDF2.loc[indicesToKeep, 'PC1']
               , ncdDF2.loc[indicesToKeep, 'PC2']
               , c = color
               , s = 50)
ax.legend(targets)
ax.grid()


loading_scores3 = pd.Series(cdpca.components_[0], index= ncd.columns)
sorted_loading_scores3 = loading_scores3.abs().sort_values(ascending=False)
top10Features3 = sorted_loading_scores3[0:10].index.values
print(loading_scores3[top10Features3])

fips         -0.614235
state        -0.613872
county       -0.492145
population    0.051024
land area    -0.032462
water area   -0.003529
date         -0.000000
dtype: float64


life


nlife = life.copy()
nlife = nlife[nlife['state'].isin(['california', 'florida', 'south dakota', 'wyoming'])]
nlife.reset_index(drop=True,inplace=True)
nlife


nnlife = nlife.copy()
for col in nnlife.columns:
    if nnlife[col].dtype == 'object':
        nnlife[col] = pd.factorize(nnlife[col])[0]


lfscaled = pd.DataFrame(StandardScaler().fit_transform(nnlife), columns= nlife.columns)


lifepca = PCA(n_components=5).fit(lfscaled)
per_var4 = np.round(lifepca.explained_variance_ratio_ * 100, decimals= 1)
labels4 = ['PC' + str(x) for x in range(1,len(per_var4)+1)]
lifePcaDF = pd.DataFrame(lifepca.transform(lfscaled), columns = labels4)
lifePcaDF


lifePcaDF = pd.concat([nlife[['state']], lifePcaDF], axis = 1)
lifePcaDF


fig = plt.figure(figsize = (8,8))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('Principal Component 1', fontsize = 15)
ax.set_ylabel('Principal Component 2', fontsize = 15)
ax.set_title('2 component PCA - Life Expectancy ', fontsize = 20)
targets = ['california', 'florida', 'south dakota', 'wyoming', np.nan]
colors = ['r', 'g', 'b','y', 'black']
for target, color in zip(targets,colors):
    indicesToKeep = lifePcaDF['state'] == target
    ax.scatter(lifePcaDF.loc[indicesToKeep, 'PC1']
               , lifePcaDF.loc[indicesToKeep, 'PC2']
               , c = color
               , s = 50)
ax.legend(targets)
ax.grid()


loading_scores5 = pd.Series(lifepca.components_[0], index= nlife.columns)
sorted_loading_scores5 = loading_scores5.abs().sort_values(ascending=False)
top10Features5 = sorted_loading_scores5[0:10].index.values
print(loading_scores5[top10Features5])

rank (male)                                          -0.301151
white female life expectancy (years)                  0.298200
male life expectancy (years)                          0.292480
white male life expectancy (years)                    0.289211
female life expectancy change 1987 to 2007 (years)    0.288658
female life expectancy (years)                        0.286506
rank (female)                                        -0.271495
male life expectancy change 1987 to 2007 (years)      0.268578
female life expectancy change 1987 to 1997 (years)    0.251781
female life expectancy change 1997 to 2007 (years)    0.246571
dtype: float64


topDF = pd.merge(states[['fips', 'state', 'county']],
                 pd.merge(nlife[['fips', 'county', 'state', 'rank (male)', 'white female life expectancy (years)', 'male life expectancy (years)', 'white male life expectancy (years)', 'female life expectancy change 1987 to 2007 (years)']],
                         fcomplete[['fips', 'state', 'households_2019', 'employed_2015','civilian_labor_force_2016', 'civilian_labor_force_2017', 'civilian_labor_force_2015']]))
topDF


ntopDF = topDF.copy()
for col in ntopDF.columns:
    if ntopDF[col].dtype == 'object':
        ntopDF[col] = pd.factorize(ntopDF[col])[0]
ntopDF = pd.DataFrame(StandardScaler().fit_transform(ntopDF), columns=ntopDF.columns)
ntopDF


toppca = PCA(n_components=10).fit(ntopDF)
per_var6 = np.round(toppca.explained_variance_ratio_ * 100, decimals= 1)
labels6 = ['PC' + str(x) for x in range(1,len(per_var6)+1)]
topPcaDF = pd.DataFrame(toppca.transform(ntopDF), columns = labels6)
topPcaDF


topPcaDF = pd.concat([topDF[['state']], topPcaDF], axis = 1)
topPcaDF


loading_scores6 = pd.Series(toppca.components_[0], index= topDF.columns)
sorted_loading_scores6 = loading_scores6.abs().sort_values(ascending=False)
top10Features6 = sorted_loading_scores6[0:10].index.values
print(loading_scores6[top10Features6])

households_2019                                       0.390954
employed_2015                                         0.389889
civilian_labor_force_2016                             0.389676
civilian_labor_force_2017                             0.389675
civilian_labor_force_2015                             0.389549
female life expectancy change 1987 to 2007 (years)    0.239753
state                                                -0.182816
county                                               -0.180209
fips                                                 -0.167646
white male life expectancy (years)                    0.166015
dtype: float64


bottom10Features2 = sorted_loading_scores6[-10:].index.values
print(loading_scores6[bottom10Features2])

civilian_labor_force_2017                             0.389675
civilian_labor_force_2015                             0.389549
female life expectancy change 1987 to 2007 (years)    0.239753
state                                                -0.182816
county                                               -0.180209
fips                                                 -0.167646
white male life expectancy (years)                    0.166015
male life expectancy (years)                          0.148770
white female life expectancy (years)                  0.141845
rank (male)                                          -0.135653
dtype: float64


fig = plt.figure(figsize = (15,15))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('Principal Component 1', fontsize = 15)
ax.set_ylabel('Principal Component 2', fontsize = 15)
ax.set_title('2 component PCA - Section B ', fontsize = 20)
targets = ['california', 'florida', 'south dakota', 'wyoming']
colors = ['r', 'g', 'b','y']
for target, color in zip(targets,colors):
    indicesToKeep = topPcaDF['state'] == target
    ax.scatter(topPcaDF.loc[indicesToKeep, 'PC1']
               , topPcaDF.loc[indicesToKeep, 'PC2']
               , c = color
               , s = 50)
ax.legend(targets)
ax.grid()


tsne = TSNE()
stsne = tsne.fit_transform(ntopDF)
stsne

C:\Users\lover\anaconda3\lib\site-packages\sklearn\manifold\_t_sne.py:783: FutureWarning: The default initialization in TSNE will change from 'random' to 'pca' in 1.2.
  FutureWarning,
C:\Users\lover\anaconda3\lib\site-packages\sklearn\manifold\_t_sne.py:793: FutureWarning: The default learning rate in TSNE will change from 200.0 to 'auto' in 1.2.
  FutureWarning,

array([[ 58.56721 , -34.618355],
       [ 58.568516, -34.62335 ],
       [ 58.570873, -34.63745 ],
       ...,
       [-27.063814,  66.90498 ],
       [-27.859016,  65.35779 ],
       [-29.66391 ,  62.852104]], dtype=float32)


stsne = pd.DataFrame(stsne, columns=['PC1', 'PC2'])
stsne = pd.concat([stsne,topDF[['state']]], axis = 1)
stsne


fig = plt.figure(figsize = (8,8))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('Principal Component 1', fontsize = 15)
ax.set_ylabel('Principal Component 2', fontsize = 15)
ax.set_title('2 component TSNE', fontsize = 20)
targets = ['california', 'florida', 'south dakota', 'wyoming']
colors = ['r', 'g', 'b','y']
for target, color in zip(targets,colors):
    indicesToKeep = stsne['state'] == target
    ax.scatter(stsne.loc[indicesToKeep, 'PC1']
               , stsne.loc[indicesToKeep, 'PC2']
               , c = color
               , s = 50)
ax.legend(targets)
ax.grid()


# show the us election results dataframes
print("House")
display(house.head())
print("President")
display(president.head())
print("Senate")
display(senate.head())

House

President

Senate


houseg = house.groupby(['state', 'year'], as_index=False).sum()
houseg


houseg.head(20)


houseg.columns

Index(['state', 'year', 'state_fips', 'state_cen', 'state_ic', 'district',
       'special', 'writein', 'candidatevotes', 'totalvotes', 'unofficial',
       'version', 'fusion_ticket'],
      dtype='object')


houseg = houseg[['state', 'candidatevotes']]
houseg


print(type(houseg))

<class 'pandas.core.frame.DataFrame'>


compg = complete.groupby(['state'], as_index=False).agg({'pop2010': 'sum', 'age_under_18_2010': 'mean'})
compg


compg['can_vote_2010'] = compg['pop2010'] * (1 -compg['age_under_18_2010']/100)
compg


print(type(compg))

<class 'pandas.core.frame.DataFrame'>


newtest = house.groupby(['state','year'], as_index=False).agg({'candidatevotes': 'sum'})
newtest = newtest.groupby('year').get_group((2010))
newtest


fullh = pd.merge(compg, newtest, on=['state'], how='outer')
fullh


fullh.head(20)


fullh['voter_turnout_2010'] = fullh['candidatevotes']/fullh['can_vote_2010']
fullh


x = fullh.drop(columns=['voter_turnout_2010'])
y = fullh[['state', 'voter_turnout_2010']]
display(x)
display(y)


x_test = x[x['state'].isin(['california', 'florida', 'south dakota', 'wyoming'])]
y_test = y[y['state'].isin(['california', 'florida', 'south dakota', 'wyoming'])]
x_train = x[~x['state'].isin(['california', 'florida', 'south dakota', 'wyoming'])]
y_train = y[~y['state'].isin(['california', 'florida', 'south dakota', 'wyoming'])]
display(x_train)
display(y_train)
'''
x_test = x.get(x['state'].isin(['california', 'florida', 'south dakota', 'wyoming']))
x_test
'''

"\nx_test = x.get(x['state'].isin(['california', 'florida', 'south dakota', 'wyoming']))\nx_test\n"


fig = plt.figure(figsize = (12,12))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('State', fontsize = 15)
ax.set_ylabel('Voter Turnout 2010', fontsize = 15)
ax.set_title('State Vs. Voter Turn out 2010', fontsize = 20, color='cornflowerblue')
targets = ['california', 'florida', 'south dakota', 'wyoming']
colors = ['r', 'g', 'b','y']
for target, color in zip(targets,colors):
    indicesToKeep = fullh['state'] == target
    ax.scatter(fullh.loc[indicesToKeep, 'state']
               , fullh.loc[indicesToKeep, 'voter_turnout_2010']
               , c = color
               , s = 50)
ax.legend(targets)
ax.grid()


for col in x_train.columns:
    if x_train[col].dtype == 'object':
        x_train[col] = pd.factorize(x_train[col])[0]
        x_test[col] = pd.factorize(x_test[col])[0]
x_train.dropna(inplace=True)
x_test.dropna(inplace=True)
for col in y_train.columns:
    if y_train[col].dtype == 'object':
        y_train[col] = pd.factorize(y_train[col])[0]
        y_test[col] = pd.factorize(y_test[col])[0]
y_test.dropna(inplace=True)
y_train.dropna(inplace=True)
display(x_test)
display(y_train)

C:\Users\lover\anaconda3\lib\site-packages\pandas\util\_decorators.py:311: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  return func(*args, **kwargs)


import time
start = time.time()
dtreg = DecisionTreeRegressor(max_depth=10)
dtreg.fit(x_train, y_train['voter_turnout_2010'])
dtrPreds = dtreg.predict(x_test)
mse = mean_squared_error(y_test['voter_turnout_2010'], dtrPreds)
print("Decision Tree Regressor MSE:",mse," with a run time of ", time.time()-start,' seconds')

Decision Tree Regressor MSE: 0.0018329825128221636  with a run time of  0.007996082305908203  seconds


start = time.time()
adreg = AdaBoostRegressor(n_estimators=200,random_state=20)
adreg.fit(x_train, y_train['voter_turnout_2010'])
adrPreds = adreg.predict(x_test)
mse = mean_squared_error(y_test['voter_turnout_2010'], adrPreds)
print("Adaboost Regressor MSE:",mse," with a run time of ", time.time()-start,' seconds')

Adaboost Regressor MSE: 0.0014155571564051278  with a run time of  0.5939323902130127  seconds


start = time.time()
rfreg = RandomForestRegressor(n_estimators=200,random_state=20)
rfreg.fit(x_train, y_train['voter_turnout_2010'])
rfrPreds = rfreg.predict(x_test)
mse = mean_squared_error(y_test['voter_turnout_2010'], rfrPreds)
print("Random Forest Regressor MSE:",mse," with a run time of ", time.time()-start,' seconds')

Random Forest Regressor MSE: 0.001843402735859305  with a run time of  0.6360979080200195  seconds


start = time.time()
gbreg = GradientBoostingRegressor(n_estimators=200,random_state=20)
gbreg.fit(x_train, y_train['voter_turnout_2010'])
gbrPreds = gbreg.predict(x_test)
mse = mean_squared_error(y_test['voter_turnout_2010'], gbrPreds)
print("Gradientboost Regressor MSE:",mse," with a run time of ", time.time()-start,' seconds')

Gradientboost Regressor MSE: 0.0012963654826801318  with a run time of  0.13191962242126465  seconds


houseg12 = house.groupby(['state', 'year'], as_index=False).agg({'candidatevotes':'sum'})
houseg12 = houseg12.groupby('year').get_group((2012))
houseg12


for col in complete.columns:
    if str(2010) in col or str(2012) in col or str(2008) in col :
        print(col)

smoking_ban_2010
pop2010
pop2012
age_under_5_2010
age_under_18_2010
age_over_65_2010
female_2010
white_2010
black_2010
native_2010
asian_2010
pac_isl_2010
two_plus_races_2010
hispanic_2010
white_not_hispanic_2010
no_move_in_one_plus_year_2010
foreign_born_2010
foreign_spoken_at_home_2010
hs_grad_2010
bachelors_2010
veterans_2010
mean_work_travel_2010
housing_units_2010
homeownership_2010
housing_multi_unit_2010
median_val_owner_occupied_2010
households_2010
persons_per_household_2010
per_capita_income_2010
median_household_income_2010
building_permits_2010
area_2010
density_2010
poverty_2010
civilian_labor_force_2008
employed_2008
unemployed_2008
unemployment_rate_2008
civilian_labor_force_2010
employed_2010
unemployed_2010
unemployment_rate_2010
civilian_labor_force_2012
employed_2012
unemployed_2012
unemployment_rate_2012


for col in life.columns:
    print(col)

fips
state
county
year
male life expectancy (years)
years behind international frontier (male)
female life expectancy (years)
years behind international frontier (female)
white male life expectancy (years)
white female life expectancy (years)
black male life expectancy (years)
black female life expectancy (years)
closest ranked countries for male life expectancy (higher)
closest ranked countries for female life expectancy (higher)
closest ranked countries for male life expectancy (lower)
closest ranked countries for female life expectancy (lower)
rank (male)
rank (female)
male life expectancy change 1987 to 2007 (years)
female life expectancy change 1987 to 2007 (years)
male life expectancy change 1987 to 1997 (years)
female life expectancy change 1987 to 1997 (years)
male life expectancy change 1997 to 2007 (years)
female life expectancy change 1997 to 2007 (years)


compg12 = complete.groupby(['state'], as_index=False).agg({'pop2010':'sum', 'pop2012':'sum', 'age_under_18_2010': 'mean',
                                                          'hs_grad_2010': 'mean', 'age_over_65_2010':'mean'})
#compg12 = compg12[['state', 'pop2010', 'age_under_18_2010', 'hs_grad_2010', 'age_over_65_2010', 'pop2012']]
compg12.head(40)


popg = countyData.groupby(['state'], as_index=False).agg({'population': 'sum'})
popg


fullh12 = pd.merge(popg, compg12)
fullh12


fullh12 = pd.concat([fullh12, fullh[['candidatevotes', 'can_vote_2010', 'voter_turnout_2010']]], axis = 1)
fullh12.rename(columns={'population':'pop1990'}, inplace=True)
fullh12.dropna(inplace=True)
fullh12.reset_index(drop=True, inplace=True)
fullh12


newtest12 = house.groupby(['state','year'], as_index=False).agg({'candidatevotes': 'sum'})
newtest12 = newtest12.groupby('year').get_group((2012))
newtest12


newtest12.reset_index(drop=True, inplace=True)
newtest12.rename(columns={'candidatevotes': 'candidatevotes_2012'}, inplace=True)
fullh12.rename(columns={'candidatevotes': 'candidatevotes_2010'},inplace=True)
fullh12 = pd.concat([fullh12, newtest12[['candidatevotes_2012']]], axis = 1)
fullh12


fullh12.dropna(inplace=True)
fullh12


fullh12['multi_ratio'] = (fullh12['hs_grad_2010']/100 * fullh12['age_under_18_2010']/100) / fullh12['age_over_65_2010'] + 1
fullh12


fullh12['can_vote_2012'] = fullh12['multi_ratio'] * fullh12['can_vote_2010']
fullh12['voter_turnout_2012'] = fullh12['candidatevotes_2012'] / fullh12['can_vote_2012']
fullh12


fig = plt.figure(figsize = (12,12))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('State', fontsize = 15)
ax.set_ylabel('Voter Turnout 2012', fontsize = 15)
ax.set_title('State Vs. Voter Turn out 2012', fontsize = 20, color='cornflowerblue')
targets = ['california', 'florida', 'south dakota', 'wyoming']
colors = ['r', 'g', 'b','y']
for target, color in zip(targets,colors):
    indicesToKeep = fullh12['state'] == target
    ax.scatter(fullh12.loc[indicesToKeep, 'state']
               , fullh12.loc[indicesToKeep, 'voter_turnout_2012']
               , c = color
               , s = 50)
ax.legend(targets)
ax.grid()


fig = plt.figure(figsize = (12,12))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('State', fontsize = 15)
ax.set_ylabel('Voter Turnout 2010', fontsize = 15)
ax.set_title('State Vs. Voter Turn out 2010', fontsize = 20, color='cornflowerblue')
targets = ['california', 'florida', 'south dakota', 'wyoming']
colors = ['r', 'g', 'b','y']
for target, color in zip(targets,colors):
    indicesToKeep = fullh12['state'] == target
    ax.scatter(fullh12.loc[indicesToKeep, 'state']
               , fullh12.loc[indicesToKeep, 'voter_turnout_2010']
               , c = color
               , s = 50)
ax.legend(targets)
ax.grid()


nfullh12 = fullh12.copy()
nfullh12['state'] = pd.factorize(nfullh12['state'])[0]


nfullh12


x10 = nfullh12.drop(columns=['voter_turnout_2010'])
x12 = nfullh12.drop(columns=['voter_turnout_2012'])
y10 = nfullh12['voter_turnout_2010']
y12 = nfullh12['voter_turnout_2012']


x10_train, x10_test, y10_train, y10_test = train_test_split(x10, y10, test_size=0.2, random_state=20)
x12_train, x12_test, y12_train, y12_test = train_test_split(x12, y12, test_size=0.2, random_state=20)


start = time.time()
dtr10 = DecisionTreeRegressor(random_state=40)
# define parameter grid
parameters_grid = {
    'max_depth': [7, 10, 12],
    'min_samples_split': [2, 4],
    'min_samples_leaf': [2, 8]
}

# define grid search
grid_search = GridSearchCV(estimator=dtr10, param_grid=parameters_grid, cv=10)

# fit estimator
grid_search.fit(x10_train, y10_train)

# get best estimator
best = grid_search.best_estimator_
print(best)

# print best parameters
print(pd.DataFrame.from_dict(grid_search.best_params_, orient='index', columns=['Selected Value']).T)

dtr10preds = best.predict(x10_test)
mse10 = mean_squared_error(y10_test, dtr10preds)
run10 = time.time()-start
print("Decision Tree Regressor for 2010 cycle, MSE: ", mse10, "with run time of ",run10, ' seconds.')

DecisionTreeRegressor(max_depth=7, min_samples_leaf=8, random_state=40)
                max_depth  min_samples_leaf  min_samples_split
Selected Value          7                 8                  2
Decision Tree Regressor for 2010 cycle, MSE:  0.001800649267498328 with run time of  1.0939931869506836  seconds.


plt.figure(figsize=(15, 15))
tr = tree.plot_tree(best, feature_names=x10_train.columns, filled=True)


importancedtr10 = best.feature_importances_
# plot feature importance
plt.bar([x for x in range(len(importancedtr10))], importancedtr10)
plt.show()


start = time.time()
dtr12 = DecisionTreeRegressor(random_state=40)
# define parameter grid
parameters_grid = {
    'max_depth': [7,10, 12 ],
    'min_samples_split': [2, 4],
    'min_samples_leaf': [2, 8]
}

# define grid search
grid_search = GridSearchCV(estimator=dtr12, param_grid=parameters_grid, cv=10)

# fit estimator
grid_search.fit(x12_train, y12_train)

# get best estimator
bestdtr12 = grid_search.best_estimator_
print(bestdtr12)

# print best parameters
print(pd.DataFrame.from_dict(grid_search.best_params_, orient='index', columns=['Selected Value']).T)

dtr12preds = bestdtr12.predict(x12_test)
mse12 = mean_squared_error(y12_test, dtr12preds)
run12 = time.time()-start
print("Decision Tree Regressor for 2012 cycle, MSE: ", mse12, "with run time of ",run12, ' seconds.')

DecisionTreeRegressor(max_depth=7, min_samples_leaf=8, random_state=40)
                max_depth  min_samples_leaf  min_samples_split
Selected Value          7                 8                  2
Decision Tree Regressor for 2012 cycle, MSE:  0.0011470368785370265 with run time of  1.1189322471618652  seconds.


plt.figure(figsize=(15, 15))
tr = tree.plot_tree(bestdtr12, feature_names=x12_train.columns, filled=True)


importancedtr12 = bestdtr12.feature_importances_
# plot feature importance
plt.bar([x for x in range(len(importancedtr12))], importancedtr12)
plt.show()


start = time.time()
rfr10 = RandomForestRegressor(random_state=40)

# define parameter grid
parameters_grid = {
    'max_depth': [2, 3],
    'min_samples_split': [2, 4],
    'min_samples_leaf': [2, 8],
    'n_estimators': [100, 150]
}

# define grid search
grid_search = GridSearchCV(estimator=rfr10, param_grid=parameters_grid, cv=10)

# fit estimator
grid_search.fit(x10_train, y10_train)

# get best estimator
bestrfr10 = grid_search.best_estimator_
print(bestrfr10)

# print best parameters
print(pd.DataFrame.from_dict(grid_search.best_params_, orient='index', columns=['Selected Value']).T)

rfr10preds = bestrfr10.predict(x10_test)
mse10 = mean_squared_error(y10_test, rfr10preds)
run10 = time.time()-start
print("Random Forest Regressor for 2010 cycle, MSE: ", mse10, "with run time of ",run10, ' seconds.')

RandomForestRegressor(max_depth=2, min_samples_leaf=8, random_state=40)
                max_depth  min_samples_leaf  min_samples_split  n_estimators
Selected Value          2                 8                  2           100
Random Forest Regressor for 2010 cycle, MSE:  0.0017958517212420182 with run time of  55.026798248291016  seconds.


importancerfr10 = bestrfr10.feature_importances_
# plot feature importance
plt.bar([x for x in range(len(importancerfr10))], importancerfr10)
plt.show()


start = time.time()
rfr12 = RandomForestRegressor(random_state=40)

# define parameter grid
parameters_grid = {
    'max_depth': [2, 3],
    'min_samples_split': [2, 4],
    'min_samples_leaf': [2, 8],
    'n_estimators': [100, 150]
}

# define grid search
grid_search = GridSearchCV(estimator=rfr12, param_grid=parameters_grid, cv=10)

# fit estimator
grid_search.fit(x12_train, y12_train)

# get best estimator
bestrfr12 = grid_search.best_estimator_
print(bestrfr12)

# print best parameters
print(pd.DataFrame.from_dict(grid_search.best_params_, orient='index', columns=['Selected Value']).T)

rfr12preds = bestrfr12.predict(x12_test)
mse12 = mean_squared_error(y12_test, rfr12preds)
run12 = time.time()-start
print("Random Forest Regressor for 2012 cycle, MSE: ", mse12, "with run time of ",run12, ' seconds.')

RandomForestRegressor(max_depth=2, min_samples_leaf=8, random_state=40)
                max_depth  min_samples_leaf  min_samples_split  n_estimators
Selected Value          2                 8                  2           100
Random Forest Regressor for 2012 cycle, MSE:  0.0012061475209400269 with run time of  59.686084270477295  seconds.


importancerfr12 = bestrfr12.feature_importances_
# plot feature importance
plt.bar([x for x in range(len(importancerfr12))], importancerfr12)
plt.show()


start = time.time()
adr10 = AdaBoostRegressor(random_state=40)

# define parameter grid
parameters_grid = {
    'learning_rate': [0,0.5,1],
    'n_estimators': [100, 150]
}

# define grid search
grid_search = GridSearchCV(estimator=adr10, param_grid=parameters_grid, cv=10)

# fit estimator
grid_search.fit(x10_train, y10_train)

# get best estimator
bestadr10 = grid_search.best_estimator_
print(bestadr10)

# print best parameters
print(pd.DataFrame.from_dict(grid_search.best_params_, orient='index', columns=['Selected Value']).T)

adr10preds = bestadr10.predict(x10_test)
mse10 = mean_squared_error(y10_test, adr10preds)
run10 = time.time()-start
print("Adaboost Regressor for 2010 cycle, MSE: ", mse10, "with run time of ",run10, ' seconds.')

C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_validation.py:372: FitFailedWarning: 
20 fits failed out of a total of 60.
The score on these train-test partitions for these parameters will be set to nan.
If these failures are not expected, you can try to debug them by setting error_score='raise'.

Below are more details about the failures:
--------------------------------------------------------------------------------
20 fits failed with the following error:
Traceback (most recent call last):
  File "C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_validation.py", line 680, in _fit_and_score
    estimator.fit(X_train, y_train, **fit_params)
  File "C:\Users\lover\anaconda3\lib\site-packages\sklearn\ensemble\_weight_boosting.py", line 1065, in fit
    return super().fit(X, y, sample_weight)
  File "C:\Users\lover\anaconda3\lib\site-packages\sklearn\ensemble\_weight_boosting.py", line 114, in fit
    raise ValueError("learning_rate must be greater than zero")
ValueError: learning_rate must be greater than zero

  warnings.warn(some_fits_failed_message, FitFailedWarning)
C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_search.py:972: UserWarning: One or more of the test scores are non-finite: [        nan         nan -2.78110886 -3.01305966 -2.72880065 -2.64417937]
  category=UserWarning,

AdaBoostRegressor(learning_rate=1, n_estimators=150, random_state=40)
                learning_rate  n_estimators
Selected Value              1           150
Adaboost Regressor for 2010 cycle, MSE:  0.0013454571718047037 with run time of  14.200978755950928  seconds.


importanceadr10 = bestadr10.feature_importances_
# plot feature importance
plt.bar([x for x in range(len(importanceadr10))], importanceadr10)
plt.show()


start = time.time()
adr12 = AdaBoostRegressor(random_state=40)

# define parameter grid
parameters_grid = {
    'learning_rate': [0,0.5,1],
    'n_estimators': [100, 150]
}

# define grid search
grid_search = GridSearchCV(estimator=adr12, param_grid=parameters_grid, cv=10)

# fit estimator
grid_search.fit(x12_train, y12_train)

# get best estimator
bestadr12 = grid_search.best_estimator_
print(bestadr12)

# print best parameters
print(pd.DataFrame.from_dict(grid_search.best_params_, orient='index', columns=['Selected Value']).T)

adr12preds = bestadr12.predict(x12_test)
mse12 = mean_squared_error(y12_test, adr12preds)
run12 = time.time()-start
print("Adaboost Regressor for 2012 cycle, MSE: ", mse12, "with run time of ",run12, ' seconds.')

C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_validation.py:372: FitFailedWarning: 
20 fits failed out of a total of 60.
The score on these train-test partitions for these parameters will be set to nan.
If these failures are not expected, you can try to debug them by setting error_score='raise'.

Below are more details about the failures:
--------------------------------------------------------------------------------
20 fits failed with the following error:
Traceback (most recent call last):
  File "C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_validation.py", line 680, in _fit_and_score
    estimator.fit(X_train, y_train, **fit_params)
  File "C:\Users\lover\anaconda3\lib\site-packages\sklearn\ensemble\_weight_boosting.py", line 1065, in fit
    return super().fit(X, y, sample_weight)
  File "C:\Users\lover\anaconda3\lib\site-packages\sklearn\ensemble\_weight_boosting.py", line 114, in fit
    raise ValueError("learning_rate must be greater than zero")
ValueError: learning_rate must be greater than zero

  warnings.warn(some_fits_failed_message, FitFailedWarning)
C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_search.py:972: UserWarning: One or more of the test scores are non-finite: [        nan         nan -0.08162522 -0.07681493 -0.11986831 -0.13357404]
  category=UserWarning,

AdaBoostRegressor(learning_rate=0.5, n_estimators=150, random_state=40)
                learning_rate  n_estimators
Selected Value            0.5         150.0
Random Forest Regressor for 2012 cycle, MSE:  0.0014433330194855497 with run time of  15.569398880004883  seconds.


importanceadr12 = bestadr12.feature_importances_
# plot feature importance
plt.bar([x for x in range(len(importanceadr12))], importanceadr12)
plt.show()


start = time.time()
gbr10 = GradientBoostingRegressor(random_state=40)

# define parameter grid
parameters_grid = {
    'learning_rate': [0,0.5,1],
    'n_estimators': [100, 150]
}

# define grid search
grid_search = GridSearchCV(estimator=gbr10, param_grid=parameters_grid, cv=10)

# fit estimator
grid_search.fit(x10_train, y10_train)

# get best estimator
bestgbr10 = grid_search.best_estimator_
print(bestgbr10)

# print best parameters
print(pd.DataFrame.from_dict(grid_search.best_params_, orient='index', columns=['Selected Value']).T)

gbr10preds = bestgbr10.predict(x10_test)
mse10 = mean_squared_error(y10_test, gbr10preds)
run10 = time.time()-start
print("GradientBoost Regressor for 2010 cycle, MSE: ", mse10, "with run time of ",run10, ' seconds.')

GradientBoostingRegressor(learning_rate=0.5, random_state=40)
                learning_rate  n_estimators
Selected Value            0.5         100.0
GradientBoost Regressor for 2010 cycle, MSE:  0.0019462548968943766 with run time of  3.213634490966797  seconds.

C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_validation.py:372: FitFailedWarning: 
20 fits failed out of a total of 60.
The score on these train-test partitions for these parameters will be set to nan.
If these failures are not expected, you can try to debug them by setting error_score='raise'.

Below are more details about the failures:
--------------------------------------------------------------------------------
20 fits failed with the following error:
Traceback (most recent call last):
  File "C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_validation.py", line 680, in _fit_and_score
    estimator.fit(X_train, y_train, **fit_params)
  File "C:\Users\lover\anaconda3\lib\site-packages\sklearn\ensemble\_gb.py", line 525, in fit
    self._check_params()
  File "C:\Users\lover\anaconda3\lib\site-packages\sklearn\ensemble\_gb.py", line 275, in _check_params
    "learning_rate must be greater than 0 but was %r" % self.learning_rate
ValueError: learning_rate must be greater than 0 but was 0

  warnings.warn(some_fits_failed_message, FitFailedWarning)
C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_search.py:972: UserWarning: One or more of the test scores are non-finite: [         nan          nan  -5.7963179   -5.7963179  -12.80302701
 -12.80302701]
  category=UserWarning,


importancegbr10 = bestgbr10.feature_importances_
# plot feature importance
plt.bar([x for x in range(len(importancegbr10))], importancegbr10)
plt.show()


start = time.time()
gbr12 = GradientBoostingRegressor(random_state=40)

# define parameter grid
parameters_grid = {
    'learning_rate': [0,0.5,1],
    'n_estimators': [100, 150]
}

# define grid search
grid_search = GridSearchCV(estimator=gbr12, param_grid=parameters_grid, cv=10)

# fit estimator
grid_search.fit(x12_train, y12_train)

# get best estimator
bestgbr12 = grid_search.best_estimator_
print(bestgbr12)

# print best parameters
print(pd.DataFrame.from_dict(grid_search.best_params_, orient='index', columns=['Selected Value']).T)

gbr12preds = bestgbr12.predict(x12_test)
mse12 = mean_squared_error(y12_test, gbr12preds)
run12 = time.time()-start
print("GradientBoost Regressor for 2012 cycle, MSE: ", mse12, "with run time of ",run12, ' seconds.')

GradientBoostingRegressor(learning_rate=0.5, random_state=40)
                learning_rate  n_estimators
Selected Value            0.5         100.0
GradientBoost Regressor for 2012 cycle, MSE:  0.002364375245120338 with run time of  2.9606430530548096  seconds.

C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_validation.py:372: FitFailedWarning: 
20 fits failed out of a total of 60.
The score on these train-test partitions for these parameters will be set to nan.
If these failures are not expected, you can try to debug them by setting error_score='raise'.

Below are more details about the failures:
--------------------------------------------------------------------------------
20 fits failed with the following error:
Traceback (most recent call last):
  File "C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_validation.py", line 680, in _fit_and_score
    estimator.fit(X_train, y_train, **fit_params)
  File "C:\Users\lover\anaconda3\lib\site-packages\sklearn\ensemble\_gb.py", line 525, in fit
    self._check_params()
  File "C:\Users\lover\anaconda3\lib\site-packages\sklearn\ensemble\_gb.py", line 275, in _check_params
    "learning_rate must be greater than 0 but was %r" % self.learning_rate
ValueError: learning_rate must be greater than 0 but was 0

  warnings.warn(some_fits_failed_message, FitFailedWarning)
C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_search.py:972: UserWarning: One or more of the test scores are non-finite: [        nan         nan -0.44096898 -0.44096898 -0.76539187 -0.76539187]
  category=UserWarning,


importancegbr12 = bestgbr12.feature_importances_
# plot feature importance
plt.bar([x for x in range(len(importancegbr12))], importancegbr12)
plt.show()


house


clsh = house.groupby(['state', 'year','party'], as_index=False).agg({'candidatevotes': 'sum'})
clsh = clsh[~clsh['state'].isin(['district of columbia'])]
clsh.reset_index(drop=True, inplace=True)
#clsh = clsh.sort_values(['candidatevotes'], ascending=False).groupby(['state', 'year'], as_index=False)
clsh.head(19)


print(type(clsh))

<class 'pandas.core.frame.DataFrame'>


fclsh = clsh.copy()
#fclsh = fclsh.sort_values(['candidatevotes'], ascending=False)
fclsh['label'] = fclsh.groupby(['state', 'year'], as_index=False)['candidatevotes'].transform('sum')
fclsh


nfc = pd.DataFrame(columns=fclsh.columns)
for st in fclsh.state.unique():
    for yr in fclsh.year.unique():
        temp = fclsh.groupby(['state', 'year']).get_group((st, yr))
        temp['label'] = temp.loc[(temp['candidatevotes'] == temp['candidatevotes'].max())]['party']
        nfc = pd.concat([nfc, temp], axis = 0)
nfc

C:\Users\lover\anaconda3\lib\site-packages\ipykernel_launcher.py:5: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  """


nfc.dropna(inplace=True)
nfc.reset_index(drop=True, inplace=True)
nfc


nfc['label'].replace({'democrat': 'D', 'republican': 'R', 'democratic-farmer-labor': 'D', 'independent': 'R'},inplace=True)
nfc


nfc['label'].unique()

array(['D', 'R', 'I'], dtype=object)


from sklearn.svm import SVC
svmnfc = nfc.copy()
svmX = svmnfc.drop(columns=['label'])
svmY = svmnfc['label']
for col in svmX:
    if svmX[col].dtype == 'object':
        svmX[col] = pd.factorize(svmX[col])[0]
svmX = pd.DataFrame(StandardScaler().fit_transform(svmX), columns=svmX.columns)
svmX


svmnfc = pd.concat([svmX, svmY], axis = 1)
svmnfc


svX_train, svX_test, svY_train, svY_test = train_test_split(svmX, svmY, random_state=40, test_size=0.2)


start = time.time()
svm = SVC(random_state=40)
# define parameter grid
parameters_grid = {
    'gamma': ['auto', 1, 2],
    'shrinking': [False, True],
    'probability': [False, True],
    'tol': [0.0005, 0.00075, 0.001]
}

# define grid search
grid_search = GridSearchCV(estimator=svm, param_grid=parameters_grid, cv=10)

# fit estimator
grid_search.fit(svX_train, svY_train)

# get best estimator
bestsvm = grid_search.best_estimator_
print(bestsvm)

# print best parameters
print(pd.DataFrame.from_dict(grid_search.best_params_, orient='index', columns=['Selected Value']).T)

svmpreds = bestsvm.predict(svX_test)
sacc = accuracy_score(svY_test, svmpreds)
runs = time.time()-start
print("SVM Classifier, Accuracy: ", sacc, "with run time of ",runs, ' seconds.')

C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_split.py:680: UserWarning: The least populated class in y has only 6 members, which is less than n_splits=10.
  UserWarning,

SVC(gamma='auto', random_state=40, shrinking=False, tol=0.0005)
               gamma probability shrinking     tol
Selected Value  auto       False     False  0.0005
SVM Classifier, Accuracy:  1.0 with run time of  35.58691954612732  seconds.


importancesvm = bestsvm.feature_names_in_
# plot feature importance
plt.bar([x for x in range(len(importancesvm))], importancesvm)
plt.show()


start = time.time()
dtc = DecisionTreeClassifier(random_state=40)
# define parameter grid
parameters_grid = {
    'max_depth': [7, 10, 12],
    'min_samples_split': [2, 4],
    'min_samples_leaf': [2, 8]
}

# define grid search
grid_search = GridSearchCV(estimator=dtc, param_grid=parameters_grid, cv=10)

# fit estimator
grid_search.fit(svX_train, svY_train)

# get best estimator
bestdtc = grid_search.best_estimator_
print(bestdtc)

# print best parameters
print(pd.DataFrame.from_dict(grid_search.best_params_, orient='index', columns=['Selected Value']).T)

dtcpreds = bestdtc.predict(svX_test)
cacc = accuracy_score(svY_test, dtcpreds)
runc = time.time()-start
print("Decision Tree Classifier, Accuracy: ", cacc, "with run time of ",runc, ' seconds.')

C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_split.py:680: UserWarning: The least populated class in y has only 6 members, which is less than n_splits=10.
  UserWarning,

DecisionTreeClassifier(max_depth=7, min_samples_leaf=2, random_state=40)
                max_depth  min_samples_leaf  min_samples_split
Selected Value          7                 2                  2
Decision Tree Classifier, Accuracy:  1.0 with run time of  1.2300403118133545  seconds.


plt.figure(figsize=(15, 15))
tr = tree.plot_tree(bestdtc, feature_names=svmX.columns, class_names=['D', 'R', 'I'], filled=True)


importancedtc = bestdtc.feature_importances_
# plot feature importance
plt.bar([x for x in range(len(importancedtc))], importancedtc)
plt.show()


start = time.time()
rfc = RandomForestClassifier(random_state=40)

# define parameter grid
parameters_grid = {
    'max_depth': [2, 3],
    'min_samples_split': [2, 4],
    'min_samples_leaf': [2, 8],
    'n_estimators': [100, 150]
}

# define grid search
grid_search = GridSearchCV(estimator=rfc, param_grid=parameters_grid, cv=10)

# fit estimator
grid_search.fit(svX_train, svY_train)

# get best estimator
bestrfc = grid_search.best_estimator_
print(bestrfc)

# print best parameters
print(pd.DataFrame.from_dict(grid_search.best_params_, orient='index', columns=['Selected Value']).T)

rfcpreds = bestrfc.predict(svX_test)
accrfc = accuracy_score(svY_test, rfcpreds)
runrfc = time.time()-start
print("Random Forest Classifier , Accuracy: ", accrfc, "with run time of ",runrfc, ' seconds.')


importancerfc = bestrfc.feature_importances_
# plot feature importance
plt.bar([x for x in range(len(importancerfc))], importancerfc)
plt.show()


start = time.time()
adc = AdaBoostClassifier(random_state=40)

# define parameter grid
parameters_grid = {
    'learning_rate': [0,0.5,1],
    'n_estimators': [100, 150]
}

# define grid search
grid_search = GridSearchCV(estimator=adc, param_grid=parameters_grid, cv=10)

# fit estimator
grid_search.fit(svX_train, svY_train)

# get best estimator
bestadc = grid_search.best_estimator_
print(bestadc)

# print best parameters
print(pd.DataFrame.from_dict(grid_search.best_params_, orient='index', columns=['Selected Value']).T)

adcpreds = bestadc.predict(svX_test)
accadc = accuracy_score(svY_test, adcpreds)
runadc = time.time()-start
print("Adaboost Classifier , Accuracy: ", accadc, "with run time of ",runadc, ' seconds.')

C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_split.py:680: UserWarning: The least populated class in y has only 6 members, which is less than n_splits=10.
  UserWarning,
C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_validation.py:372: FitFailedWarning: 
20 fits failed out of a total of 60.
The score on these train-test partitions for these parameters will be set to nan.
If these failures are not expected, you can try to debug them by setting error_score='raise'.

Below are more details about the failures:
--------------------------------------------------------------------------------
20 fits failed with the following error:
Traceback (most recent call last):
  File "C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_validation.py", line 680, in _fit_and_score
    estimator.fit(X_train, y_train, **fit_params)
  File "C:\Users\lover\anaconda3\lib\site-packages\sklearn\ensemble\_weight_boosting.py", line 486, in fit
    return super().fit(X, y, sample_weight)
  File "C:\Users\lover\anaconda3\lib\site-packages\sklearn\ensemble\_weight_boosting.py", line 114, in fit
    raise ValueError("learning_rate must be greater than zero")
ValueError: learning_rate must be greater than zero

  warnings.warn(some_fits_failed_message, FitFailedWarning)
C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_search.py:972: UserWarning: One or more of the test scores are non-finite: [nan nan  1.  1.  1.  1.]
  category=UserWarning,

AdaBoostClassifier(learning_rate=0.5, n_estimators=100, random_state=40)
                learning_rate  n_estimators
Selected Value            0.5         100.0
Adaboost Classifier , Accuracy:  1.0 with run time of  26.840372562408447  seconds.


importanceadc = bestadc.feature_importances_
# plot feature importance
plt.bar([x for x in range(len(importanceadc))], importanceadc)
plt.show()


start = time.time()
gbc = GradientBoostingClassifier(random_state=40)

# define parameter grid
parameters_grid = {
    'learning_rate': [0,0.5,1],
    'n_estimators': [100, 150]
}

# define grid search
grid_search = GridSearchCV(estimator=gbc, param_grid=parameters_grid, cv=10)

# fit estimator
grid_search.fit(svX_train, svY_train)

# get best estimator
bestgbc = grid_search.best_estimator_
print(bestgbc)

# print best parameters
print(pd.DataFrame.from_dict(grid_search.best_params_, orient='index', columns=['Selected Value']).T)

gbcpreds = bestgbc.predict(svX_test)
accgbc = accuracy_score(svY_test, gbcpreds)
rungbc = time.time()-start
print("Gradientboost Classifier , Accuracy: ", accgbc, "with run time of ",rungbc, ' seconds.')

C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_split.py:680: UserWarning: The least populated class in y has only 6 members, which is less than n_splits=10.
  UserWarning,
C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_validation.py:372: FitFailedWarning: 
20 fits failed out of a total of 60.
The score on these train-test partitions for these parameters will be set to nan.
If these failures are not expected, you can try to debug them by setting error_score='raise'.

Below are more details about the failures:
--------------------------------------------------------------------------------
20 fits failed with the following error:
Traceback (most recent call last):
  File "C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_validation.py", line 680, in _fit_and_score
    estimator.fit(X_train, y_train, **fit_params)
  File "C:\Users\lover\anaconda3\lib\site-packages\sklearn\ensemble\_gb.py", line 525, in fit
    self._check_params()
  File "C:\Users\lover\anaconda3\lib\site-packages\sklearn\ensemble\_gb.py", line 275, in _check_params
    "learning_rate must be greater than 0 but was %r" % self.learning_rate
ValueError: learning_rate must be greater than 0 but was 0

  warnings.warn(some_fits_failed_message, FitFailedWarning)
C:\Users\lover\anaconda3\lib\site-packages\sklearn\model_selection\_search.py:972: UserWarning: One or more of the test scores are non-finite: [nan nan  1.  1.  1.  1.]
  category=UserWarning,

GradientBoostingClassifier(learning_rate=0.5, random_state=40)
                learning_rate  n_estimators
Selected Value            0.5         100.0
Gradientboost Classifier , Accuracy:  1.0 with run time of  22.804075002670288  seconds.


importancegbc = bestadc.feature_importances_
# plot feature importance
plt.bar([x for x in range(len(importancegbc))], importancegbc)
plt.show()


nfc10 = nfc.groupby(['year']).get_group((2010))


fig = plt.figure(figsize = (12,12))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('State', fontsize = 15)
ax.set_ylabel('Label', fontsize = 15)
ax.set_title('State Vs. Label in 2010', fontsize = 20, color='cornflowerblue')
targets = ['california', 'florida', 'south dakota', 'wyoming']
colors = ['r', 'g', 'b','y']
for target, color in zip(targets,colors):
    indicesToKeep = nfc10['state'] == target
    ax.scatter(nfc10.loc[indicesToKeep, 'state']
               , nfc10.loc[indicesToKeep, 'label']
               , c = color
               , s = 50)
ax.legend(targets)
ax.grid()


nfc12 = nfc.groupby(['year']).get_group((2012))
fig = plt.figure(figsize = (12,12))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('State', fontsize = 15)
ax.set_ylabel('Label', fontsize = 15)
ax.set_title('State Vs. Label in 2012', fontsize = 20, color='cornflowerblue')
targets = ['california', 'florida', 'south dakota', 'wyoming']
colors = ['r', 'g', 'b','y']
for target, color in zip(targets,colors):
    indicesToKeep = nfc12['state'] == target
    ax.scatter(nfc12.loc[indicesToKeep, 'state']
               , nfc12.loc[indicesToKeep, 'label']
               , c = color
               , s = 50)
ax.legend(targets)
ax.grid()

	FIPS	State	County	Land Area
0	1001	Alabama	Autauga	1543.7
1	1003	Alabama	Baldwin	4135.0
2	1005	Alabama	Barbour	2292.1
3	1007	Alabama	Bibb	1611.9
4	1009	Alabama	Blount	1672.3
...	...	...	...	...
3106	56037	Wyoming	Sweetwater	27003.0
3107	56039	Wyoming	Teton	10380.6
3108	56041	Wyoming	Uinta	5391.7
3109	56043	Wyoming	Washakie	5802.0
3110	56045	Wyoming	Weston	6210.6

	FIPS	State	County	Water Area
0	1001	Alabama	Autauga	22.0
1	1003	Alabama	Baldwin	1115.1
2	1005	Alabama	Barbour	50.7
3	1007	Alabama	Bibb	8.1
4	1009	Alabama	Blount	12.9
...	...	...	...	...
3106	56037	Wyoming	Sweetwater	170.0
3107	56039	Wyoming	Teton	554.3
3108	56041	Wyoming	Uinta	15.3
3109	56043	Wyoming	Washakie	7.0
3110	56045	Wyoming	Weston	5.7

	fips	state	county	land area	water area	date	population
0	1001	alabama	autauga	1543.7	22.0	1990	34222
1	1003	alabama	baldwin	4135.0	1115.1	1990	98280
2	1005	alabama	barbour	2292.1	50.7	1990	25417
3	1007	alabama	bibb	1611.9	8.1	1990	16576
4	1009	alabama	blount	1672.3	12.9	1990	39248
5	1011	alabama	bullock	1618.9	2.7	1990	11042
6	1013	alabama	butler	2012.3	2.7	1990	21892
7	1015	alabama	calhoun	1576.0	10.0	1990	116034
8	1017	alabama	chambers	1547.3	14.9	1990	36876
9	1019	alabama	cherokee	1432.7	121.3	1990	19543

	Date	Value
0	2016-04-15	17897.46
1	2016-04-14	17926.43
2	2016-04-13	17908.28
3	2016-04-12	17721.25
4	2016-04-11	17556.41
...	...	...
32596	1896-07-18	31.50
32597	1896-07-17	31.94
32598	1896-07-16	31.95
32599	1896-07-15	32.28
32600	1896-07-14	33.43

	Table C1. Median Household Income by County: 1969, 1979, 1989, and 1999	Unnamed: 1	Unnamed: 2	Unnamed: 3	Unnamed: 4	Unnamed: 5
0	NaN	NaN	NaN	NaN	NaN	NaN
1	(In current dollars)	NaN	NaN	NaN	NaN	NaN
2	NaN	NaN	NaN	NaN	NaN	NaN
3	NaN	1999	1989	1979	1969	NaN
4	NaN	NaN	NaN	NaN	NaN	NaN
...	...	...	...	...	...	...
3267	(301) 457-3242	NaN	NaN	NaN	NaN	NaN
3268	NaN	NaN	NaN	NaN	NaN	NaN
3269	NaN	NaN	NaN	NaN	NaN	NaN
3270	NaN	NaN	NaN	NaN	NaN	NaN
3271	NaN	---------	NaN	NaN	NaN	NaN

Importing packages and libraries¶

Section A - Exploration and Preproccesing¶

CountyData¶

EconomicData¶

GeneralDemographicData¶

LifeExpectancyData¶

USElectionResults19762020¶

Summary of Section A:¶

Section B - Deminsional Reduction¶

Preparations for dimensionality reduction¶

First Algorithm - PCA (Principal Components Analysis):¶

PCA results overview:¶

PCA overview :¶

Second Algorithm T-SNE¶

TSNE overview:¶

Section C - Regression:¶

1- Decision Tree Regressor :¶

2- Random Forest Regressor:¶

- 3 Adaboost Regressor:¶

4- Gradientboost Regressor:¶

Section D (Classification):¶

1- SVM Classifier:¶

2- Decision Tree Classifier:¶

3- Random Forest Classifier :¶

4- Adaboost Classifier:¶

5- Gradientboost Classifier:¶

Section E (Bonus) :¶

	day	month	year	value
0	15	4	2016	17897.46
1	14	4	2016	17926.43
2	13	4	2016	17908.28
3	12	4	2016	17721.25
4	11	4	2016	17556.41
...	...	...	...	...
32596	18	7	1896	31.50
32597	17	7	1896	31.94
32598	16	7	1896	31.95
32599	15	7	1896	32.28
32600	14	7	1896	33.43

	median household by county	mhs1999	mhs1989	mhs1979	mhs1969
5	United States	41,994	30,056	16,841	8,486
6	NaN	NaN	NaN	NaN	NaN
7	Alabama	34,135	23,597	13,669	6,419
8	Autauga County, AL	42,013	28,337	16,524	6,808
9	Baldwin County, AL	40,250	25,712	14,614	6,430
...	...	...	...	...	...
3254	Sweetwater County, WY	46,537	36,210	24,114	8,284
3255	Teton County, WY	54,614	31,586	18,442	8,410
3256	Uinta County, WY	44,544	33,259	22,584	8,409
3257	Washakie County, WY	34,943	25,172	18,239	7,588
3258	Weston County, WY	32,348	26,213	20,021	7,796

	median household by county	mhs1999	mhs1989	mhs1979	mhs1969
5	United States	41994	30056	16841	8486
6	0	0	0	0	0
7	Alabama	34135	23597	13669	6419
8	Autauga County, AL	42013	28337	16524	6808
9	Baldwin County, AL	40250	25712	14614	6430
...	...	...	...	...	...
3254	Sweetwater County, WY	46537	36210	24114	8284
3255	Teton County, WY	54614	31586	18442	8410
3256	Uinta County, WY	44544	33259	22584	8409
3257	Washakie County, WY	34943	25172	18239	7588
3258	Weston County, WY	32348	26213	20021	7796

	Date	Price	Open	High	Low	Vol.	Change %
0	Dec 31, 2019	28,538.44	28,414.64	28,547.35	28,376.49	193.34M	0.27%
1	Dec 30, 2019	28,462.14	28,654.76	28,664.69	28,428.98	185.07M	-0.64%
2	Dec 27, 2019	28,645.26	28,675.34	28,701.66	28,608.98	184.93M	0.08%
3	Dec 26, 2019	28,621.39	28,539.46	28,624.10	28,535.15	155.97M	0.37%
4	Dec 24, 2019	28,515.45	28,572.57	28,576.80	28,503.21	95.29M	-0.13%
...	...	...	...	...	...	...	...
2761	Jan 09, 2009	8,599.18	8,738.80	8,800.45	8,541.75	-	-1.64%
2762	Jan 08, 2009	8,742.46	8,769.94	8,807.14	8,593.52	-	-0.31%
2763	Jan 07, 2009	8,769.70	8,996.94	8,996.94	8,690.45	-	-2.72%
2764	Jan 06, 2009	9,015.10	8,954.57	9,175.19	8,868.07	-	0.69%
2765	Jan 05, 2009	8,952.89	9,027.13	9,093.47	8,841.70	-	-0.91%

	day	month	year	price	open	high	low	vol.	change %
0	31	12	2019	28,538.44	28,414.64	28,547.35	28,376.49	193.34M	0.27%
1	30	12	2019	28,462.14	28,654.76	28,664.69	28,428.98	185.07M	-0.64%
2	27	12	2019	28,645.26	28,675.34	28,701.66	28,608.98	184.93M	0.08%
3	26	12	2019	28,621.39	28,539.46	28,624.10	28,535.15	155.97M	0.37%
4	24	12	2019	28,515.45	28,572.57	28,576.80	28,503.21	95.29M	-0.13%
...	...	...	...	...	...	...	...	...	...
2761	9	1	2009	8,599.18	8,738.80	8,800.45	8,541.75	-	-1.64%
2762	8	1	2009	8,742.46	8,769.94	8,807.14	8,593.52	-	-0.31%
2763	7	1	2009	8,769.70	8,996.94	8,996.94	8,690.45	-	-2.72%
2764	6	1	2009	9,015.10	8,954.57	9,175.19	8,868.07	-	0.69%
2765	5	1	2009	8,952.89	9,027.13	9,093.47	8,841.70	-	-0.91%

	day	month	year	price	open	high	low	vol. (millions)	change %
0	31.0	12.0	2019.0	28538.44	28414.64	28547.35	28376.49	193.34	0.27
1	30.0	12.0	2019.0	28462.14	28654.76	28664.69	28428.98	185.07	0.00
2	27.0	12.0	2019.0	28645.26	28675.34	28701.66	28608.98	184.93	0.08
3	26.0	12.0	2019.0	28621.39	28539.46	28624.10	28535.15	155.97	0.37
4	24.0	12.0	2019.0	28515.45	28572.57	28576.80	28503.21	95.29	0.00
...	...	...	...	...	...	...	...	...	...
2761	9.0	1.0	2009.0	8599.18	8738.80	8800.45	8541.75	0.00	0.00
2762	8.0	1.0	2009.0	8742.46	8769.94	8807.14	8593.52	0.00	0.00
2763	7.0	1.0	2009.0	8769.70	8996.94	8996.94	8690.45	0.00	0.00
2764	6.0	1.0	2009.0	9015.10	8954.57	9175.19	8868.07	0.00	0.69
2765	5.0	1.0	2009.0	8952.89	9027.13	9093.47	8841.70	0.00	0.00

	Unnamed: 0	Unnamed: 1	Unnamed: 2	Unnamed: 3	Unnamed: 4	Unnamed: 5	Unnamed: 6	Unnamed: 7	Unnamed: 8
0	FIPS	Countyname	Postal	LineCode	IndustryName	Gross domestic product (GDP) by county	NaN	NaN	NaN
1	NaN	NaN	NaN	NaN	NaN	(thousands of dollars)	NaN	NaN	NaN
2	NaN	NaN	NaN	NaN	NaN	2012	2013	2014	2015
3	01001	Autauga	AL	1	All Industries	1383941	1363368	1402516	1539406
4	01001	Autauga	AL	2	Private goods-producing industries	286396	310468	323582	346355
...	...	...	...	...	...	...	...	...	...
12454	56045	Weston	WY	4	Government and government enterprises	54750	58758	59301	59461
12455	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
12456	Source: U.S. Bureau of Economic Analysis	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
12457	(D) Not shown to avoid disclosure of confident...	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
12458	Note: Detail may not sum to higher-level aggre...	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN

	fips	state	name	pop2000	pop2010	pop2011	pop2012	pop2013	pop2014	pop2015	...	poverty_under_18_2019	two_plus_races_2019	unemployment_rate_2019	uninsured_2019	uninsured_65_and_older_2019	uninsured_under_19_2019	uninsured_under_6_2019	veterans_2019	white_2019	white_not_hispanic_2019
0	1001	Alabama	Autauga County	43671.0	54571	55199.0	54927.0	54695.0	54864.0	54838.0	...	23.2	2.2	3.5	7.1	0.0	1.7	1.7	12.6	76.8	74.6
1	1003	Alabama	Baldwin County	140415.0	182265	186534.0	190048.0	194736.0	199064.0	202863.0	...	13.4	1.7	4.0	8.9	0.3	3.8	2.2	11.8	86.2	83.1
2	1005	Alabama	Barbour County	29038.0	27457	27351.0	27175.0	26947.0	26749.0	26264.0	...	50.1	1.2	9.4	11.3	0.3	3.3	3.4	6.6	46.8	45.8
3	1007	Alabama	Bibb County	20826.0	22915	22745.0	22658.0	22503.0	22533.0	22561.0	...	NaN	0.6	7.0	10.7	0.0	2.0	4.5	8.0	76.8	74.5
4	1009	Alabama	Blount County	51024.0	57322	57562.0	57595.0	57623.0	57546.0	57590.0	...	18.4	1.6	3.1	10.8	0.2	5.9	6.1	7.7	95.5	86.9
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
3137	56037	Wyoming	Sweetwater County	37613.0	43806	44013.0	45042.0	45145.0	44981.0	44732.0	...	NaN	2.3	5.7	11.3	0.5	9.2	13.8	8.6	93.4	79.6
3138	56039	Wyoming	Teton County	18251.0	21294	21476.0	21709.0	22326.0	22817.0	23029.0	...	NaN	0.7	0.7	12.7	0.0	10.1	5.9	5.3	89.3	81.3
3139	56041	Wyoming	Uinta County	19742.0	21118	20899.0	20999.0	20960.0	20845.0	20780.0	...	NaN	3.5	5.5	11.2	0.6	6.8	1.8	7.4	93.4	87.5
3140	56043	Wyoming	Washakie County	8289.0	8533	8460.0	8421.0	8427.0	8288.0	8296.0	...	NaN	3.8	4.1	15.0	1.5	7.0	7.8	11.9	89.7	81.9
3141	56045	Wyoming	Weston County	6644.0	7208	7141.0	7074.0	7136.0	7142.0	7181.0	...	NaN	1.3	4.0	11.8	0.0	8.6	7.1	10.3	97.4	96.4

	state	name	smoking_ban_2010
0	Alabama	Autauga County	none
1	Alabama	Baldwin County	none
2	Alabama	Barbour County	partial
3	Alabama	Bibb County	none
4	Alabama	Blount County	none
...	...	...	...
3137	Wyoming	Sweetwater County	none
3138	Wyoming	Teton County	partial
3139	Wyoming	Uinta County	none
3140	Wyoming	Washakie County	none
3141	Wyoming	Weston County	none

	fips	pop2000	pop2010	pop2011	pop2012	pop2013	pop2014	pop2015	pop2016	pop2017	...	poverty_under_18_2019	two_plus_races_2019	unemployment_rate_2019	uninsured_2019	uninsured_65_and_older_2019	uninsured_under_19_2019	uninsured_under_6_2019	veterans_2019	white_2019	white_not_hispanic_2019
0	1001.0	43671.0	54571.0	55199.0	54927.0	54695.0	54864.0	54838.0	55278.0	55504.0	...	23.200000	2.2	3.5	7.1	0.0	1.7	1.7	12.6	76.8	74.6
1	1003.0	140415.0	182265.0	186534.0	190048.0	194736.0	199064.0	202863.0	207509.0	212628.0	...	13.400000	1.7	4.0	8.9	0.3	3.8	2.2	11.8	86.2	83.1
2	1005.0	29038.0	27457.0	27351.0	27175.0	26947.0	26749.0	26264.0	25774.0	25270.0	...	50.100000	1.2	9.4	11.3	0.3	3.3	3.4	6.6	46.8	45.8
3	1007.0	20826.0	22915.0	22745.0	22658.0	22503.0	22533.0	22561.0	22633.0	22668.0	...	34.113333	0.6	7.0	10.7	0.0	2.0	4.5	8.0	76.8	74.5
4	1009.0	51024.0	57322.0	57562.0	57595.0	57623.0	57546.0	57590.0	57562.0	58013.0	...	18.400000	1.6	3.1	10.8	0.2	5.9	6.1	7.7	95.5	86.9
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
3137	56037.0	37613.0	43806.0	44013.0	45042.0	45145.0	44981.0	44732.0	44245.0	43534.0	...	20.110000	2.3	5.7	11.3	0.5	9.2	13.8	8.6	93.4	79.6
3138	56039.0	18251.0	21294.0	21476.0	21709.0	22326.0	22817.0	23029.0	23180.0	23265.0	...	20.320000	0.7	0.7	12.7	0.0	10.1	5.9	5.3	89.3	81.3
3139	56041.0	19742.0	21118.0	20899.0	20999.0	20960.0	20845.0	20780.0	20711.0	20495.0	...	25.250000	3.5	5.5	11.2	0.6	6.8	1.8	7.4	93.4	87.5
3140	56043.0	8289.0	8533.0	8460.0	8421.0	8427.0	8288.0	8296.0	8188.0	8064.0	...	33.856667	3.8	4.1	15.0	1.5	7.0	7.8	11.9	89.7	81.9
3141	56045.0	6644.0	7208.0	7141.0	7074.0	7136.0	7142.0	7181.0	7198.0	6927.0	...	34.710000	1.3	4.0	11.8	0.0	8.6	7.1	10.3	97.4	96.4

	gdp2012	gdp2013	gdp2014	gdp2015
3	1383941	1363368	1402516	1539406
4	286396	310468	323582	346355
5	948490	904599	928438	1037309
6	149055	148301	150496	155742
7	5599194	6365080	6547396	6436107
...	...	...	...	...
12450	62263	64489	65430	65738
12451	332472	306384	312816	347555
12452	181482	149558	148903	175708
12453	96240	98069	104612	112386
12454	54750	58758	59301	59461

	state	name	smoking_ban_2010	fips	pop2000	pop2010	pop2011	pop2012	pop2013	pop2014	...	poverty_under_18_2019	two_plus_races_2019	unemployment_rate_2019	uninsured_2019	uninsured_65_and_older_2019	uninsured_under_19_2019	uninsured_under_6_2019	veterans_2019	white_2019	white_not_hispanic_2019
1465	mississippi	smith county	none	28129.0	16182.0	16491.0	16531.0	16342.0	16233.0	16211.0	...	34.476667	0.4	4.2	14.0	0.0	6.3	4.8	6.6	75.8	74.0
116	arkansas	bradley county	none	5011.0	12600.0	11508.0	11425.0	11262.0	11120.0	11007.0	...	33.200000	0.2	7.5	11.0	0.0	4.6	6.0	9.3	65.9	55.5
1102	kentucky	todd county	comprehensive	21219.0	11971.0	12460.0	12410.0	12597.0	12450.0	12409.0	...	35.673333	1.6	6.7	15.2	1.6	17.4	15.9	7.6	87.5	85.7
1918	north carolina	davidson county	none	37057.0	147246.0	162878.0	162972.0	163112.0	163398.0	163530.0	...	23.600000	1.8	5.7	11.3	0.6	5.8	3.6	7.9	85.8	80.2
107	arizona	pinal county	comprehensive	4021.0	179727.0	375770.0	378226.0	381910.0	384258.0	394004.0	...	17.500000	4.2	6.4	8.8	0.7	7.5	7.5	10.5	79.9	56.8
273	colorado	jackson county	comprehensive	8057.0	1577.0	1394.0	1358.0	1326.0	1337.0	1385.0	...	35.460000	0.4	2.0	6.1	0.0	7.0	3.8	14.2	97.9	82.2
2968	washington	jefferson county	none	53031.0	25953.0	29872.0	29835.0	29782.0	29990.0	30135.0	...	28.290000	3.8	5.5	5.4	0.2	3.8	0.3	13.8	90.9	88.4
725	indiana	hamilton county	none	18057.0	182740.0	274569.0	283273.0	289599.0	296789.0	302828.0	...	5.300000	2.4	2.4	4.4	0.3	3.4	3.8	6.0	86.8	83.7
2442	tennessee	cocke county	none	47029.0	33565.0	35662.0	35401.0	35455.0	35325.0	35201.0	...	40.000000	2.1	8.5	11.2	0.8	5.0	1.4	10.0	94.8	93.3
260	colorado	denver county	partial	8031.0	554636.0	600158.0	619356.0	633798.0	648049.0	663271.0	...	18.200000	3.8	3.6	9.3	1.4	4.2	2.3	5.2	76.1	54.2
2982	washington	skamania county	comprehensive	53059.0	9872.0	11066.0	11150.0	11198.0	11308.0	11376.0	...	33.906667	2.3	4.2	4.0	0.3	0.0	0.0	12.3	92.8	87.7
1113	louisiana	acadia parish	partial	22001.0	58861.0	61773.0	61826.0	61984.0	62284.0	62664.0	...	35.600000	2.8	7.3	10.6	0.6	5.0	3.9	5.8	79.3	77.4
1839	new york	cortland county	comprehensive	36023.0	48599.0	49336.0	49373.0	49024.0	48912.0	48742.0	...	18.200000	1.9	4.9	3.2	0.4	3.0	4.2	7.2	94.6	92.6
394	georgia	bartow county	none	13015.0	76019.0	100157.0	100213.0	100365.0	101056.0	101397.0	...	18.200000	2.4	4.5	14.9	0.8	8.7	8.6	7.8	83.7	77.5
2805	utah	weber county	partial	49057.0	196533.0	231236.0	233890.0	236342.0	238222.0	240219.0	...	12.500000	3.5	3.3	8.9	0.7	6.4	5.8	7.7	89.2	76.1
764	indiana	randolph county	none	18135.0	27401.0	26171.0	26021.0	25848.0	25610.0	25313.0	...	31.396667	2.5	5.1	9.4	0.5	6.1	7.7	8.3	94.9	92.8
2215	oregon	curry county	none	41015.0	21137.0	22364.0	22477.0	22249.0	22217.0	22130.0	...	28.160000	6.3	7.5	6.1	0.4	2.6	3.8	15.4	90.9	86.3
1295	michigan	ogemaw county	none	26129.0	21645.0	21699.0	21495.0	21365.0	21167.0	20961.0	...	28.843333	1.3	7.4	6.5	0.3	6.3	7.9	10.6	96.5	94.6
1947	north carolina	madison county	comprehensive	37115.0	19635.0	20764.0	20853.0	20902.0	21171.0	21219.0	...	35.683333	1.6	4.7	9.3	0.3	5.5	2.4	8.7	95.4	94.0
885	iowa	woodbury county	comprehensive	19193.0	103877.0	102172.0	102622.0	102290.0	102195.0	102200.0	...	17.300000	3.5	4.1	5.7	0.5	1.9	0.5	7.0	85.7	73.0

	Black male life expectancy (years)	Black female life expectancy (years)	Closest ranked countries for male life expectancy (higher)	Closest ranked countries for female life expectancy (higher)
0	NaN	73.4	Chile,Guadeloupe,Panama,United Arab Emirates,V...	Austria,Costa Rica,Netherlands Antilles,New Ze...
1	64.3	73.3	Bahrain,Chile,Guam,Panama,United Arab Emirates	Guadeloupe,Ireland,Israel,Malta,Netherlands An...
2	64.7	73.5	Belize,Chile,Portugal,Puerto Rico,United Arab ...	Ireland,Israel,Malta,Netherlands Antilles,Port...
3	64.4	73.6	Bahrain,Belize,Chile,Guam,Puerto Rico	Costa Rica,Denmark,Ireland,Malta,Portugal
4	64.5	73.5	Bahrain,Belize,Chile,Guam,Portugal	Denmark,Ireland,Malta,Mayotte,Reunion
...	...	...	...	...
66082	NaN	NaN	Belgium,Denmark,Guadeloupe,Luxembourg,Virgin I...	Costa Rica,Ireland,Slovenia,United Kingdom,Uni...
66083	NaN	NaN	Belgium,Cuba,Kuwait,Luxembourg,Martinique	Costa Rica,Greece,Ireland,Slovenia,United States
66084	NaN	NaN	Denmark,Finland,Guadeloupe,Kuwait,Virgin Islan...	Channel Islands,Costa Rica,Greece,Malta,United...
66085	NaN	NaN	Denmark,Finland,Guadeloupe,Kuwait,Virgin Islan...	Channel Islands,Costa Rica,Malta,United Kingdo...
66086	NaN	NaN	Costa Rica,Denmark,Finland,Guadeloupe,Kuwait	Chile,Netherlands,Portugal,Slovenia,Virgin Isl...

	Black male life expectancy (years)	Black female life expectancy (years)	Closest ranked countries for male life expectancy (higher)	Closest ranked countries for female life expectancy (higher)
0	0.0	73.4	Chile,Guadeloupe,Panama,United Arab Emirates,V...	Austria,Costa Rica,Netherlands Antilles,New Ze...
1	64.3	73.3	Bahrain,Chile,Guam,Panama,United Arab Emirates	Guadeloupe,Ireland,Israel,Malta,Netherlands An...
2	64.7	73.5	Belize,Chile,Portugal,Puerto Rico,United Arab ...	Ireland,Israel,Malta,Netherlands Antilles,Port...
3	64.4	73.6	Bahrain,Belize,Chile,Guam,Puerto Rico	Costa Rica,Denmark,Ireland,Malta,Portugal
4	64.5	73.5	Bahrain,Belize,Chile,Guam,Portugal	Denmark,Ireland,Malta,Mayotte,Reunion
...	...	...	...	...
66082	0.0	0.0	Belgium,Denmark,Guadeloupe,Luxembourg,Virgin I...	Costa Rica,Ireland,Slovenia,United Kingdom,Uni...
66083	0.0	0.0	Belgium,Cuba,Kuwait,Luxembourg,Martinique	Costa Rica,Greece,Ireland,Slovenia,United States
66084	0.0	0.0	Denmark,Finland,Guadeloupe,Kuwait,Virgin Islan...	Channel Islands,Costa Rica,Greece,Malta,United...
66085	0.0	0.0	Denmark,Finland,Guadeloupe,Kuwait,Virgin Islan...	Channel Islands,Costa Rica,Malta,United Kingdo...
66086	0.0	0.0	Costa Rica,Denmark,Finland,Guadeloupe,Kuwait	Chile,Netherlands,Portugal,Slovenia,Virgin Isl...

	fips	State	County	Year	Male life expectancy (years)	Years behind international frontier (male)	Female life expectancy (years)	Years behind international frontier (female)	White male life expectancy (years)	White female life expectancy (years)	...	Closest ranked countries for male life expectancy (lower)	Closest ranked countries for female life expectancy (lower)	Rank (male)	Rank (female)	Male life expectancy change 1987 to 2007 (years)	Female life expectancy change 1987 to 2007 (years)	Male life expectancy change 1987 to 1997 (years)	Female life expectancy change 1987 to 1997 (years)	Male life expectancy change 1997 to 2007 (years)	Female life expectancy change 1997 to 2007 (years)
65515	55133	WISCONSIN	WAUKESHA	2003	77.8	1	81.7	8	77.8	81.7	...	Canada,Israel,Italy,Macao Special Administrati...	Cyprus,Germany,Guadeloupe,Macao Special Admini...	26	94	4.7	2.5	2.8	1.7	1.9	0.8
17124	19051	IOWA	DAVIS	1996	73.3	13	80.2	9	73.3	80.2	...	Brunei Darussalam,Denmark,Guadeloupe,Ireland,V...	Austria,Cyprus,Germany,Luxembourg,New Zealand	1196	732	2.6	1.7	1.6	1.3	1.0	0.4
43381	39041	OHIO	DELAWARE	2003	76.7	5	80.8	12	76.8	80.9	...	France,Malta,Netherlands,Spain,United Kingdom	Costa Rica,Ireland,Slovenia,United Kingdom,Uni...	145	513	4.8	2.3	3.4	2.1	1.4	0.2
5298	8014	COLORADO	BROOMFIELD	1993	75.1	1	80.6	3	75.1	80.6	...	Australia,Canada,Cyprus,Greece,Switzerland	Guadeloupe,Macao Special Administrative Region...	67	194	3.8	1.7	2.0	1.0	1.8	0.7
29935	28047	MISSISSIPPI	HARRISON	1997	70.5	27	77.4	22	71.7	78.4	...	Korea, Republic of,Mayotte,Qatar,Reunion,Uruguay	Argentina,French Guiana,Panama,Poland,Slovakia	2657	2802	1.6	1.4	1.1	0.2	0.5	1.2
56071	48291	TEXAS	LIBERTY	1988	68.6	36	77.2	14	69.5	77.9	...	Bulgaria,Czech Republic,French Guiana,Saint Lu...	Barbados,Cuba,Mayotte,Reunion,Slovenia	2833	2729	2.5	0.0	1.4	-0.2	1.1	0.2
2028	2290	ALASKA	YUKON-KOYUKUK	1999	74.3	11	79.6	15	73.7	79.1	...	Denmark,Finland,Guadeloupe,Ireland,Virgin Isla...	Chile,Greece,Mayotte,Reunion,Slovenia	988	1451	4.3	2.1	1.8	1.1	2.5	1.0
61201	51195	VIRGINIA	WISE	1994	69.5	35	77.8	18	69.7	77.9	...	Croatia,Korea, Republic of,New Caledonia,Saint...	Barbados,Brunei Darussalam,Chile,Kuwait,Nether...	2766	2625	2.0	-0.2	1.1	0.2	0.9	-0.4
54735	48163	TEXAS	FRIO	1996	70.3	27	78.3	18	70.7	78.3	...	Croatia,Korea, Republic of,Macedonia, the Form...	Barbados,Brunei Darussalam,Denmark,Korea, Repu...	2667	2427	3.2	0.7	1.0	0.4	2.2	0.3
49247	45055	SOUTH CAROLINA	KERSHAW	1989	69.6	29	77.9	12	71.4	79.4	...	Albania,Jamaica,Macedonia, the Former Yugoslav...	Costa Rica,Denmark,Israel,Malta,Portugal	2605	2433	3.4	1.0	1.5	0.4	1.9	0.6
27554	26161	MICHIGAN	WASHTENAW	1989	73.5	5	79.1	8	74.0	79.5	...	Channel Islands,Costa Rica,Italy,Norway,Spain	Cyprus,Finland,Macao Special Administrative Re...	208	1311	5.4	3.0	3.4	1.8	2.0	1.2
27321	26139	MICHIGAN	OTTAWA	1987	73.7	2	80.1	1	73.6	80.0	...	Canada,Macao Special Administrative Region of ...	Australia,France,Hong Kong Special Administrat...	65	126	5.1	2.3	2.6	1.4	2.5	0.9
388	1037	ALABAMA	COOSA	1997	70.4	27	78.1	20	71.6	78.9	...	Albania,Macedonia, the Former Yugoslav Republi...	Brunei Darussalam,Cuba,Netherlands Antilles,Un...	2693	2582	1.7	0.0	1.7	0.8	0.0	-0.8
27477	26153	MICHIGAN	SCHOOLCRAFT	1996	73.1	14	79.2	14	73.3	79.1	...	Brunei Darussalam,Chile,Finland,Montenegro,Vir...	Ireland,Mayotte,Montenegro,Reunion,Singapore	1418	1783	3.1	1.1	1.8	0.3	1.3	0.8
29434	27173	MINNESOTA	YELLOW MEDICINE	2000	74.8	10	81.4	6	74.8	81.3	...	Belgium,Cuba,Denmark,Guadeloupe,Luxembourg	Austria,Finland,Guadeloupe,Macao Special Admin...	732	104	3.0	1.0	2.0	1.1	1.0	-0.1
16518	18177	INDIANA	WAYNE	1999	73.1	17	78.9	19	73.4	79.1	...	Bahrain,Barbados,Belize,Montenegro,Portugal	Barbados,Brunei Darussalam,Cuba,Denmark,Kuwait	1835	2087	2.4	1.3	2.1	0.5	0.3	0.8
42588	38073	NORTH DAKOTA	RANSOM	1987	72.5	8	79.9	1	72.5	79.9	...	Channel Islands,Kuwait,Martinique,Montenegro,U...	Australia,Hong Kong Special Administrative Reg...	624	190	3.4	1.4	1.2	0.6	2.2	0.8
61515	51610	VIRGINIA	FALLS CHURCH	1993	74.1	6	80.4	5	74.5	80.7	...	Kuwait,Martinique,Singapore,Spain,United Kingdom	Belgium,Guadeloupe,Macao Special Administrativ...	302	360	6.2	3.6	2.3	1.6	3.9	2.0
54722	48161	TEXAS	FREESTONE	2004	73.0	22	78.7	24	74.1	79.5	...	Albania,Czech Republic,Guam,Mexico,Panama	Argentina,Croatia,Mexico,Slovakia,United Arab ...	2141	2198	2.7	0.8	0.7	0.2	2.0	0.6
13512	17095	ILLINOIS	KNOX	1996	73.4	12	79.7	12	73.9	79.9	...	Brunei Darussalam,Denmark,Guadeloupe,Ireland,V...	Channel Islands,Costa Rica,Greece,United Kingd...	1119	1296	2.2	0.2	1.9	0.8	0.3	-0.6

		fips	male life expectancy (years)	female life expectancy (years)	white male life expectancy (years)	white female life expectancy (years)	black male life expectancy (years)	black female life expectancy (years)	rank (male)	rank (female)	male life expectancy change 1987 to 2007 (years)	female life expectancy change 1987 to 2007 (years)	male life expectancy change 1987 to 1997 (years)	female life expectancy change 1987 to 1997 (years)	male life expectancy change 1997 to 2007 (years)	female life expectancy change 1997 to 2007 (years)
county	year
abbeville	1987	45001.0	68.7	76.9	70.9	78.9	64.8	73.5	2805.0	2814.0	3.3	1.2	2.2	0.8	1.1	0.4
	1988	45001.0	68.7	77.0	71.0	79.0	64.8	73.6	2803.0	2792.0	3.3	1.2	2.2	0.8	1.1	0.4
	1989	45001.0	68.8	77.1	71.2	79.1	64.8	73.6	2818.0	2791.0	3.3	1.2	2.2	0.8	1.1	0.4
	1990	45001.0	68.8	77.1	71.2	79.2	64.7	73.6	2836.0	2798.0	3.3	1.2	2.2	0.8	1.1	0.4
	1991	45001.0	69.3	77.1	71.8	79.2	65.2	73.7	2710.0	2802.0	3.3	1.2	2.2	0.8	1.1	0.4
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
ziebach	2003	46137.0	73.2	80.0	71.4	78.7	0.0	0.0	1983.0	1092.0	2.0	0.6	1.3	0.3	0.7	0.3
	2004	46137.0	73.2	79.9	71.3	78.6	0.0	0.0	2038.0	1294.0	2.0	0.6	1.3	0.3	0.7	0.3
	2005	46137.0	72.9	80.1	70.9	78.7	0.0	0.0	2217.0	1223.0	2.0	0.6	1.3	0.3	0.7	0.3
	2006	46137.0	72.6	79.8	70.6	78.4	0.0	0.0	2353.0	1580.0	2.0	0.6	1.3	0.3	0.7	0.3
	2007	46137.0	72.9	79.9	70.7	78.4	0.0	0.0	2281.0	1576.0	2.0	0.6	1.3	0.3	0.7	0.3

	year	state	state_po	state_fips	state_cen	state_ic	office	district	stage	runoff	special	candidate	party	writein	mode	candidatevotes	totalvotes	unofficial	version	fusion_ticket
0	1976	ALABAMA	AL	1	63	41	US HOUSE	1	GEN	False	False	BILL DAVENPORT	DEMOCRAT	False	TOTAL	58906	157170	False	20220331	False
1	1976	ALABAMA	AL	1	63	41	US HOUSE	1	GEN	False	False	JACK EDWARDS	REPUBLICAN	False	TOTAL	98257	157170	False	20220331	False
2	1976	ALABAMA	AL	1	63	41	US HOUSE	1	GEN	False	False	WRITEIN	NaN	True	TOTAL	7	157170	False	20220331	False
3	1976	ALABAMA	AL	1	63	41	US HOUSE	2	GEN	False	False	J CAROLE KEAHEY	DEMOCRAT	False	TOTAL	66288	156362	False	20220331	False
4	1976	ALABAMA	AL	1	63	41	US HOUSE	2	GEN	False	False	WILLIAM L "BILL" DICKINSON	REPUBLICAN	False	TOTAL	90069	156362	False	20220331	False
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
31098	2020	WYOMING	WY	56	83	68	US HOUSE	0	GEN	False	False	LYNNETTE GREY BULL	DEMOCRAT	False	TOTAL	66576	278503	False	20220331	False
31099	2020	WYOMING	WY	56	83	68	US HOUSE	0	GEN	False	False	OVERVOTES	NaN	False	TOTAL	1274	278503	False	20220331	False
31100	2020	WYOMING	WY	56	83	68	US HOUSE	0	GEN	False	False	RICHARD BRUBAKER	LIBERTARIAN	False	TOTAL	10154	278503	False	20220331	False
31101	2020	WYOMING	WY	56	83	68	US HOUSE	0	GEN	False	False	UNDERVOTES	NaN	False	TOTAL	6337	278503	False	20220331	False
31102	2020	WYOMING	WY	56	83	68	US HOUSE	0	GEN	False	False	WRITEIN	NaN	True	TOTAL	525	278503	False	20220331	False

	year	state	state_po	state_fips	state_cen	state_ic	office	candidate	party_detailed	writein	candidatevotes	totalvotes	version	notes	party_simplified
0	1976	ALABAMA	AL	1	63	41	US PRESIDENT	CARTER, JIMMY	DEMOCRAT	False	659170	1182850	20210113	NaN	DEMOCRAT
1	1976	ALABAMA	AL	1	63	41	US PRESIDENT	FORD, GERALD	REPUBLICAN	False	504070	1182850	20210113	NaN	REPUBLICAN
2	1976	ALABAMA	AL	1	63	41	US PRESIDENT	MADDOX, LESTER	AMERICAN INDEPENDENT PARTY	False	9198	1182850	20210113	NaN	OTHER
3	1976	ALABAMA	AL	1	63	41	US PRESIDENT	BUBAR, BENJAMIN ""BEN""	PROHIBITION	False	6669	1182850	20210113	NaN	OTHER
4	1976	ALABAMA	AL	1	63	41	US PRESIDENT	HALL, GUS	COMMUNIST PARTY USE	False	1954	1182850	20210113	NaN	OTHER
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
4282	2020	WYOMING	WY	56	83	68	US PRESIDENT	JORGENSEN, JO	LIBERTARIAN	False	5768	278503	20210113	NaN	LIBERTARIAN
4283	2020	WYOMING	WY	56	83	68	US PRESIDENT	PIERCE, BROCK	INDEPENDENT	False	2208	278503	20210113	NaN	OTHER
4284	2020	WYOMING	WY	56	83	68	US PRESIDENT	NaN	NaN	True	1739	278503	20210113	NaN	OTHER
4285	2020	WYOMING	WY	56	83	68	US PRESIDENT	OVERVOTES	NaN	False	279	278503	20210113	NaN	OTHER
4286	2020	WYOMING	WY	56	83	68	US PRESIDENT	UNDERVOTES	NaN	False	1459	278503	20210113	NaN	OTHER

	year	state	state_po	state_fips	state_cen	state_ic	office	district	stage	special	candidate	party_detailed	writein	mode	candidatevotes	totalvotes	unofficial	version	party_simplified
0	1976	ARIZONA	AZ	4	86	61	US SENATE	statewide	gen	False	SAM STEIGER	REPUBLICAN	False	total	321236	741210	False	20210114	REPUBLICAN
1	1976	ARIZONA	AZ	4	86	61	US SENATE	statewide	gen	False	WM. MATHEWS FEIGHAN	INDEPENDENT	False	total	1565	741210	False	20210114	OTHER
2	1976	ARIZONA	AZ	4	86	61	US SENATE	statewide	gen	False	DENNIS DECONCINI	DEMOCRAT	False	total	400334	741210	False	20210114	DEMOCRAT
3	1976	ARIZONA	AZ	4	86	61	US SENATE	statewide	gen	False	ALLAN NORWITZ	LIBERTARIAN	False	total	7310	741210	False	20210114	LIBERTARIAN
4	1976	ARIZONA	AZ	4	86	61	US SENATE	statewide	gen	False	BOB FIELD	INDEPENDENT	False	total	10765	741210	False	20210114	OTHER
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
3624	2020	WYOMING	WY	56	83	68	US SENATE	statewide	gen	False	UNDER VOTES	NaN	False	total	6401	278503	False	20210114	OTHER
3625	2021	GEORGIA	GA	13	58	44	US SENATE	statewide	runoff	True	KELLY LOEFFLER	REPUBLICAN	False	total	2194848	4483294	True	20210114	REPUBLICAN
3626	2021	GEORGIA	GA	13	58	44	US SENATE	statewide	runoff	True	RAPHAEL WARNOCK	DEMOCRAT	False	total	2288446	4483294	True	20210114	DEMOCRAT
3627	2021	GEORGIA	GA	13	58	44	US SENATE	statewide	runoff	False	DAVID A. PERDUE	REPUBLICAN	False	total	2213979	4483241	True	20210114	REPUBLICAN
3628	2021	GEORGIA	GA	13	58	44	US SENATE	statewide	runoff	False	JON OSSOFF	DEMOCRAT	False	total	2269262	4483241	True	20210114	DEMOCRAT

	fips	state	county	land area	water area	date	population	name	smoking_ban_2010	pop2000	...	closest ranked countries for male life expectancy (lower)	closest ranked countries for female life expectancy (lower)	rank (male)	rank (female)	male life expectancy change 1987 to 2007 (years)	female life expectancy change 1987 to 2007 (years)	male life expectancy change 1987 to 1997 (years)	female life expectancy change 1987 to 1997 (years)	male life expectancy change 1997 to 2007 (years)	female life expectancy change 1997 to 2007 (years)
0	6001	california	alameda	1910.1	216.0	1990	1279182	alameda county	none	1443741.0	...	barbados,germany,ireland,netherlands antilles,...	austria,denmark,germany,united kingdom,virgin ...	1444	2262	6.1	4.2	2.6	1.9	3.5	2.3
1	6001	california	alameda	1910.1	216.0	1990	1279182	alameda county	none	1443741.0	...	aruba,austria,netherlands antilles,singapore,u...	costa rica,denmark,montenegro,new zealand,virg...	1538	2310	6.1	4.2	2.6	1.9	3.5	2.3
2	6001	california	alameda	1910.1	216.0	1990	1279182	alameda county	none	1443741.0	...	aruba,finland,guadeloupe,luxembourg,virgin isl...	costa rica,denmark,guadeloupe,montenegro,new z...	1709	2212	6.1	4.2	2.6	1.9	3.5	2.3
3	6001	california	alameda	1910.1	216.0	1990	1279182	alameda county	none	1443741.0	...	finland,guadeloupe,luxembourg,netherlands anti...	costa rica,israel,montenegro,new zealand,portugal	1547	2186	6.1	4.2	2.6	1.9	3.5	2.3
4	6001	california	alameda	1910.1	216.0	1990	1279182	alameda county	none	1443741.0	...	barbados,guadeloupe,luxembourg,netherlands ant...	costa rica,israel,montenegro,new zealand,unite...	1515	2002	6.1	4.2	2.6	1.9	3.5	2.3
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
4258	56045	wyoming	weston	6210.6	5.7	1990	6518	weston county	none	6644.0	...	brunei darussalam,chile,finland,korea, republi...	cuba,denmark,greece,mayotte,reunion	957	749	3.5	1.7	2.0	0.2	1.5	1.5
4259	56045	wyoming	weston	6210.6	5.7	1990	6518	weston county	none	6644.0	...	chile,denmark,finland,guadeloupe,virgin island...	cuba,denmark,kuwait,mayotte,reunion	779	808	3.5	1.7	2.0	0.2	1.5	1.5
4260	56045	wyoming	weston	6210.6	5.7	1990	6518	weston county	none	6644.0	...	brunei darussalam,chile,korea, republic of,por...	cuba,denmark,french guiana,mayotte,reunion	883	742	3.5	1.7	2.0	0.2	1.5	1.5
4261	56045	wyoming	weston	6210.6	5.7	1990	6518	weston county	none	6644.0	...	brunei darussalam,chile,korea, republic of,por...	cuba,denmark,greece,mayotte,reunion	843	563	3.5	1.7	2.0	0.2	1.5	1.5
4262	56045	wyoming	weston	6210.6	5.7	1990	6518	weston county	none	6644.0	...	brunei darussalam,chile,korea, republic of,por...	channel islands,greece,malta,united kingdom,un...	727	393	3.5	1.7	2.0	0.2	1.5	1.5

	fips	state	county	land area	water area	date	population	name	smoking_ban_2010	pop2000	...	closest ranked countries for male life expectancy (lower)	closest ranked countries for female life expectancy (lower)	rank (male)	rank (female)	male life expectancy change 1987 to 2007 (years)	female life expectancy change 1987 to 2007 (years)	male life expectancy change 1987 to 1997 (years)	female life expectancy change 1987 to 1997 (years)	male life expectancy change 1997 to 2007 (years)	female life expectancy change 1997 to 2007 (years)
0	-0.994517	-1.292797	-1.695422	-0.542879	0.031266	0.0	1.557546	-1.695422	-0.932869	1.591993	...	-1.283402	-1.120590	0.272777	1.270062	2.108264	2.168689	1.028268	1.886193	2.508414	1.834447
1	-0.994517	-1.292797	-1.695422	-0.542879	0.031266	0.0	1.557546	-1.695422	-0.932869	1.591993	...	-1.277623	-1.113691	0.383143	1.323672	2.108264	2.168689	1.028268	1.886193	2.508414	1.834447
2	-0.994517	-1.292797	-1.695422	-0.542879	0.031266	0.0	1.557546	-1.695422	-0.932869	1.591993	...	-1.271844	-1.106792	0.583915	1.214217	2.108264	2.168689	1.028268	1.886193	2.508414	1.834447
3	-0.994517	-1.292797	-1.695422	-0.542879	0.031266	0.0	1.557546	-1.695422	-0.932869	1.591993	...	-1.266065	-1.099893	0.393710	1.185178	2.108264	2.168689	1.028268	1.886193	2.508414	1.834447
4	-0.994517	-1.292797	-1.695422	-0.542879	0.031266	0.0	1.557546	-1.695422	-0.932869	1.591993	...	-1.260286	-1.092995	0.356139	0.979670	2.108264	2.168689	1.028268	1.886193	2.508414	1.834447
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
4258	1.536376	1.782646	1.743077	0.390691	-0.354885	0.0	-0.266271	1.743077	-0.932869	-0.284350	...	0.409871	-0.830844	-0.299011	-0.419794	0.000000	-0.034734	0.222846	-1.361937	-0.227058	0.838411
4259	1.536376	1.782646	1.743077	0.390691	-0.354885	0.0	-0.266271	1.743077	-0.932869	-0.284350	...	0.589022	-0.913628	-0.508001	-0.353897	0.000000	-0.034734	0.222846	-1.361937	-0.227058	0.838411
4260	1.536376	1.782646	1.743077	0.390691	-0.354885	0.0	-0.266271	1.743077	-0.932869	-0.284350	...	-0.104468	-0.906730	-0.385895	-0.427612	0.000000	-0.034734	0.222846	-1.361937	-0.227058	0.838411
4261	1.536376	1.782646	1.743077	0.390691	-0.354885	0.0	-0.266271	1.743077	-0.932869	-0.284350	...	-0.104468	-0.830844	-0.432859	-0.627536	0.000000	-0.034734	0.222846	-1.361937	-0.227058	0.838411
4262	1.536376	1.782646	1.743077	0.390691	-0.354885	0.0	-0.266271	1.743077	-0.932869	-0.284350	...	1.987561	-0.644579	-0.569055	-0.817407	0.000000	-0.034734	0.222846	-1.361937	-0.227058	0.838411

	PC1	PC2	PC3	PC4	PC5	PC6	PC7	PC8	PC9	PC10	PC11	PC12	PC13	PC14	PC15	PC16	PC17	PC18	PC19	PC20
0	20.078847	-3.148031	-7.355897	3.444966	-6.396216	-1.810544	0.934945	7.104521	0.704144	1.961503	-4.614112	-0.593923	1.856628	1.635280	0.697579	0.692138	-0.352161	-0.810567	0.596144	1.046048
1	20.074478	-3.124059	-7.352466	3.415184	-6.431199	-1.832794	0.929761	7.091101	0.701512	1.956618	-4.580999	-0.523129	1.788868	1.603075	0.658649	0.659437	-0.393194	-0.824488	0.557209	1.041088
2	20.072625	-3.128731	-7.343302	3.419709	-6.435340	-1.811738	0.984852	7.078249	0.701469	1.916766	-4.534777	-0.460299	1.735658	1.560922	0.598274	0.617784	-0.408606	-0.847924	0.494980	1.062441
3	20.089584	-3.217743	-7.374150	3.470125	-6.394723	-1.738035	1.112069	7.021074	0.719006	1.844027	-4.389529	-0.219927	1.653719	1.469248	0.446238	0.547608	-0.418714	-0.872242	0.425404	1.061322
4	20.103518	-3.306641	-7.390837	3.543311	-6.314850	-1.647529	1.230007	6.987476	0.741491	1.760574	-4.299363	-0.055473	1.647802	1.402108	0.308695	0.496037	-0.384707	-0.886394	0.351279	1.077347
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
4258	-4.805125	-3.782610	3.450969	0.572084	1.286960	-0.693996	-0.752482	0.088054	-2.284431	0.917229	-0.488780	2.353817	-0.026286	0.669192	-1.144722	-1.567654	1.161919	0.419524	0.434167	-0.147288
4259	-4.818992	-3.815261	3.483526	0.621961	1.215029	-0.746443	-0.694070	0.096296	-2.408408	1.134688	-0.325914	2.369398	-0.125097	0.666340	-1.406332	-1.959919	2.449471	0.366497	-0.412810	0.220758
4260	-4.755571	-3.933844	3.348024	0.732584	1.637802	-0.449386	-0.769171	0.156947	-2.161975	0.839664	-0.552770	2.368450	0.629596	0.629467	-1.824425	-1.554066	0.974357	0.132866	-0.131789	-0.214248
4261	-4.739401	-4.045749	3.329472	0.813473	1.720586	-0.339760	-0.602255	0.097849	-2.134566	0.721955	-0.426531	2.595002	0.581795	0.538655	-1.954174	-1.606795	0.977439	0.130973	-0.164600	-0.202732
4262	-4.820685	-4.018260	3.511165	0.715126	1.138003	-0.639271	-0.186445	-0.159633	-2.454920	0.907928	0.191974	3.150977	-0.891470	0.355781	-1.493693	-2.320315	3.009618	0.512628	-0.496973	0.465930

	state	name	smoking_ban_2010	fips	pop2000	pop2010	pop2011	pop2012	pop2013	pop2014	...	poverty_under_18_2019	two_plus_races_2019	unemployment_rate_2019	uninsured_2019	uninsured_65_and_older_2019	uninsured_under_19_2019	uninsured_under_6_2019	veterans_2019	white_2019	white_not_hispanic_2019
0	-1.287807	-1.696376	-0.929794	-0.995546	1.570722	1.527704	1.534377	1.546827	1.562521	1.580662	...	-1.722633	2.095480	-0.232816	-1.072087	0.319348	-0.963705	-0.782206	-1.726782	-2.245111	-1.709094
1	-1.287807	-1.679521	-0.929794	-0.995445	-0.309713	-0.328699	-0.329518	-0.330008	-0.330612	-0.331842	...	-0.161885	1.283942	2.746433	-0.294662	-0.685660	-0.621729	-1.181409	0.138051	-1.257090	-0.741204
2	-1.287807	-1.662665	-0.929794	-0.995343	-0.265533	-0.283287	-0.285151	-0.286570	-0.288115	-0.289490	...	-1.141647	1.110041	0.352394	-1.020259	-0.685660	-0.685850	-0.109864	0.890527	0.408758	0.388002
3	-1.287807	-1.645809	-0.929794	-0.995242	-0.046441	-0.059513	-0.063030	-0.064737	-0.066545	-0.067563	...	-0.792905	1.863612	0.644998	-0.761117	-0.384157	-0.728597	-0.488056	-0.352695	0.115799	0.110182
4	-1.287807	-1.628953	-0.929794	-0.995141	-0.258423	-0.274077	-0.275870	-0.277263	-0.278502	-0.280074	...	-0.345590	1.341909	0.352394	-1.072087	-0.585159	-1.262934	-1.181409	0.955960	0.627042	0.508989
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
209	1.797163	1.674795	-0.929794	1.539707	-0.262257	-0.276257	-0.277270	-0.277002	-0.277930	-0.279713	...	-0.735777	-0.339134	0.112990	0.119964	-0.183156	0.553813	1.718065	-0.123680	0.793627	0.450736
210	1.797163	1.691650	0.649380	1.539808	-0.287497	-0.303950	-0.304705	-0.305155	-0.305240	-0.306033	...	-0.712254	-1.266606	-1.217032	0.361829	-0.685660	0.746175	0.058221	-1.203320	0.558110	0.526912
211	1.797163	1.708506	-0.929794	1.539909	-0.285553	-0.304166	-0.305408	-0.306011	-0.306875	-0.308375	...	-0.160018	0.356470	0.059789	0.102688	-0.082655	0.040849	-0.803217	-0.516276	0.793627	0.804733
212	1.797163	1.725362	-0.929794	1.540011	-0.300483	-0.319647	-0.320550	-0.321187	-0.321875	-0.323287	...	0.804061	0.530371	-0.312617	0.759180	0.821852	0.083596	0.457424	0.955960	0.581087	0.553798
213	1.797163	1.742218	-0.929794	1.540112	-0.302627	-0.321277	-0.322156	-0.322812	-0.323420	-0.324648	...	0.899648	-0.918804	-0.339218	0.206344	-0.685660	0.425572	0.310349	0.432498	1.023399	1.203540

	PC1	PC2	PC3	PC4	PC5
0	18.511053	-4.096958	-7.312470	5.184999	-4.413267
1	-1.298175	0.098800	-4.032665	-1.131603	-1.959474
2	-1.566966	-3.647765	-4.036988	-4.664941	-0.265970
3	1.083668	0.507101	-4.206530	-1.225903	0.238068
4	-1.816774	-2.437155	-3.264555	-5.044697	0.212645
...	...	...	...	...	...
209	-2.038192	-2.857448	-1.047730	4.271761	2.177875
210	-0.402166	-8.248352	-4.227338	5.941364	0.131405
211	-3.093857	-1.825476	0.275134	3.270018	2.639168
212	-4.158668	-1.832448	2.339907	1.233536	2.383258
213	-4.906390	-3.313210	3.311019	0.166198	1.086813

	fips	state	county	land area	water area	date	population
0	-1.942674	-1.698598	-1.246086	-0.290689	-0.188071	0.0	-0.158631
1	-1.942541	-1.698598	-1.244089	0.489865	2.430236	0.0	0.082336
2	-1.942407	-1.698598	-1.242093	-0.065255	-0.119325	0.0	-0.191753
3	-1.942274	-1.698598	-1.240097	-0.270145	-0.221365	0.0	-0.225010
4	-1.942141	-1.698598	-1.238100	-0.251952	-0.209868	0.0	-0.139725
...	...	...	...	...	...	...	...
2936	1.730594	1.734236	2.139931	7.378187	0.166434	0.0	-0.141324
2937	1.730728	1.734236	-0.363645	2.371171	1.086949	0.0	-0.245339
2938	1.730861	1.734236	2.141928	0.868410	-0.204119	0.0	-0.217002
2939	1.730995	1.734236	2.143924	0.992001	-0.224000	0.0	-0.255811
2940	1.731128	1.734236	2.145921	1.115080	-0.227114	0.0	-0.262846

	PC1	PC2	PC3	PC4	PC5	PC6	PC7
0	2.851242	-0.381841	-0.029324	-0.091771	-0.204587	0.174538	0.0
1	2.827894	1.749168	-0.538437	-1.727197	0.012878	0.171434	0.0
2	2.839862	-0.257559	0.144689	-0.194737	-0.190478	0.173063	0.0
3	2.844112	-0.435015	0.016563	-0.122713	-0.210183	0.174337	0.0
4	2.846768	-0.365216	0.010291	-0.069592	-0.210832	0.173827	0.0
...	...	...	...	...	...	...	...
2936	-3.428057	3.458611	6.311159	-1.210326	-0.236897	-0.043185	0.0
2937	-2.042035	1.531687	1.546166	-1.062887	1.721279	-0.014380	0.0
2938	-3.220436	0.220521	0.788318	-0.053591	-0.624537	-0.000281	0.0
2939	-3.227423	0.239826	0.913031	-0.088046	-0.620369	-0.001072	0.0
2940	-3.232830	0.289512	1.023572	-0.109695	-0.615370	-0.001957	0.0

	PC1	PC2	PC3	PC4	PC5
0	2.235689	4.434735	-3.614656	2.422027	1.385105
1	2.177542	4.482998	-3.527317	2.328437	1.334714
2	2.205513	4.462234	-3.426428	2.250725	1.289718
3	2.455822	4.392868	-3.207508	2.035737	1.182018
4	2.711721	4.260365	-3.095504	1.891819	1.123355
...	...	...	...	...	...
4489	0.728112	-3.026469	1.222870	0.334806	0.131530
4490	0.702903	-3.162869	1.527003	0.855603	-1.138693
4491	1.269618	-3.301538	0.467839	-0.173868	-0.053236
4492	1.585737	-3.449993	0.673219	-0.356272	-0.075756
4493	1.173348	-3.379952	2.879027	0.645889	-1.646874

	PC1	PC2	PC3	PC4	PC5	PC6	PC7	PC8	PC9	PC10
0	4.082790	2.473945	-0.324879	1.912858	-0.478343	0.427408	0.456770	0.047968	0.123221	0.069004
1	4.076047	2.501004	-0.326608	1.887398	-0.388557	0.480950	0.496473	0.054592	0.148776	0.071614
2	4.057014	2.557305	-0.312479	1.884809	-0.176302	0.383997	0.584514	0.136168	0.153301	0.067797
3	4.167776	2.239772	-0.404217	1.761155	-0.204045	0.361948	0.555267	0.107815	0.149130	0.065696
4	4.227130	2.070157	-0.453873	1.686301	-0.138405	0.308802	0.571605	0.122374	0.148324	0.062881
...	...	...	...	...	...	...	...	...	...	...
4258	-1.176240	-2.244319	1.894844	-0.126827	0.340810	0.420539	0.014152	-0.021307	-0.077132	-0.013028
4259	-1.078254	-2.519153	1.807959	-0.248485	0.234020	0.532016	-0.042302	-0.104481	-0.058733	-0.009092
4260	-1.071265	-2.537821	1.800882	-0.275240	0.397949	0.449902	0.022114	-0.045023	-0.055961	-0.012570
4261	-0.987083	-2.782662	1.734909	-0.363761	0.535986	0.274890	0.056303	-0.008086	-0.057960	-0.019833
4262	-0.875445	-3.107016	1.646988	-0.475398	0.618340	0.111523	0.063126	0.003752	-0.062031	-0.026834

	PC1	PC2	state
0	58.567211	-34.618355	california
1	58.568516	-34.623348	california
2	58.570873	-34.637451	california
3	58.289040	-34.690372	california
4	58.094250	-34.738884	california
...	...	...	...
4258	-24.568171	69.119446	wyoming
4259	-27.116047	67.679237	wyoming
4260	-27.063814	66.904984	wyoming
4261	-27.859016	65.357788	wyoming
4262	-29.663910	62.852104	wyoming

	state	year	state_fips	state_cen	state_ic	district	special	writein	candidatevotes	totalvotes	unofficial	version	fusion_ticket
0	alabama	1976	19	1197	779	72	0	6	984181	2728482	0	384186289	0
1	alabama	1978	16	1008	656	68	0	3	642279	1438662	0	323525296	0
2	alabama	1980	23	1449	943	98	0	3	1013626	3517307	0	465067613	0
3	alabama	1982	19	1197	779	80	0	3	961019	2629352	0	384186289	0
4	alabama	1984	16	1008	656	66	0	0	1148574	2812277	0	323525296	0
...	...	...	...	...	...	...	...	...	...	...	...	...	...
1146	wyoming	2012	448	664	544	0	0	1	250700	2005600	0	161762648	0
1147	wyoming	2014	392	581	476	0	0	1	171153	1198071	0	141542317	0
1148	wyoming	2016	392	581	476	0	0	1	258788	1811516	0	141542317	0
1149	wyoming	2018	280	415	340	0	0	1	201245	1006225	0	101101655	0
1150	wyoming	2020	392	581	476	0	0	1	278503	1949521	0	141542317	0

	state	pop2010	age_under_18_2010
0	alabama	4779736.0	23.479104
1	alaska	710231.0	25.493103
2	arizona	6392017.0	25.306667
3	arkansas	2915918.0	23.542667
4	california	37253956.0	23.658621
5	colorado	5029196.0	22.057813
6	connecticut	3574097.0	22.125000
7	delaware	897934.0	22.833333
8	district of columbia	601723.0	16.800000
9	florida	18801310.0	20.776119
10	georgia	9687653.0	24.550314
11	hawaii	1360301.0	18.140000
12	idaho	1567582.0	26.511364
13	illinois	12830632.0	22.741176
14	indiana	6483802.0	24.410870
15	iowa	3046355.0	23.644444
16	kansas	2853118.0	24.143810
17	kentucky	4339367.0	23.425833
18	louisiana	4533372.0	24.717188
19	maine	1328361.0	20.337500
20	maryland	5773552.0	22.825000
21	massachusetts	6547629.0	20.821429
22	michigan	9883640.0	21.756627
23	minnesota	5303925.0	23.708046
24	mississippi	2967297.0	25.182927
25	missouri	5988927.0	23.580870
26	montana	989415.0	22.301786
27	nebraska	1826341.0	23.709677
28	nevada	2700551.0	22.952941
29	new hampshire	1316470.0	20.600000
30	new jersey	8791894.0	23.428571
31	new mexico	2059179.0	23.624242
32	new york	19378102.0	21.903226
33	north carolina	9535483.0	22.575000
34	north dakota	672591.0	21.805660
35	ohio	11536504.0	23.939773
36	oklahoma	3751351.0	24.337662
37	oregon	3831074.0	21.869444
38	pennsylvania	12702379.0	21.217910
39	rhode island	1052567.0	20.580000
40	south carolina	4625364.0	23.213043
41	south dakota	814180.0	25.281818
42	tennessee	6346105.0	22.808421
43	texas	25145561.0	25.064173
44	utah	2763885.0	31.079310
45	vermont	625741.0	20.785714
46	virginia	7994802.0	21.342105
47	washington	6724540.0	22.794872
48	west virginia	1852994.0	20.760000
49	wisconsin	5686986.0	22.684722
50	wyoming	563626.0	23.382609

	state	year	candidatevotes
17	alabama	2010	1367747
40	alaska	2010	254335
63	arizona	2010	1698145
86	arkansas	2010	774125
109	california	2010	9648096
132	colorado	2010	1763152
155	connecticut	2010	1138202
178	delaware	2010	305636
202	florida	2010	5117811
225	georgia	2010	2468680
248	hawaii	2010	360121
271	idaho	2010	447144
294	illinois	2010	3696159
317	indiana	2010	1747720
340	iowa	2010	1106591
363	kansas	2010	835529
386	kentucky	2010	1354298
409	louisiana	2010	1035948
432	maine	2010	564368
455	maryland	2010	1825472
478	massachusetts	2010	2224255
501	michigan	2010	3194901
524	minnesota	2010	2090701
547	mississippi	2010	788549
570	missouri	2010	1920675
593	montana	2010	360341
616	nebraska	2010	485546
639	nevada	2010	702788
662	new hampshire	2010	449787
685	new jersey	2010	2121584
708	new mexico	2010	596651
731	new york	2010	4753783
754	north carolina	2010	2662549
777	north dakota	2010	236344
800	ohio	2010	3825274
823	oklahoma	2010	792980
846	oregon	2010	1429356
869	pennsylvania	2010	3956401
892	rhode island	2010	335484
915	south carolina	2010	1340189
938	south dakota	2010	319426
961	tennessee	2010	1559129
984	texas	2010	4745545
1007	utah	2010	640495
1030	vermont	2010	238521
1053	virginia	2010	2189841
1076	washington	2010	2479409
1099	west virginia	2010	514373
1122	wisconsin	2010	2140482
1145	wyoming	2010	190822

	state	voter_turnout_2010
0	alabama	0.373957
1	alaska	0.480629
2	arizona	0.355676
3	arkansas	0.347230
4	california	0.339242
5	colorado	0.449799
6	connecticut	0.408936
7	delaware	0.441093
8	district of columbia	NaN
9	florida	0.343590
10	georgia	0.337745
11	hawaii	0.323401
12	idaho	0.388148
13	illinois	0.372867
14	indiana	0.356601
15	iowa	0.475736
16	kansas	0.386056
17	kentucky	0.407573
18	louisiana	0.303543
19	maine	0.533325
20	maryland	0.409690
21	massachusetts	0.429035
22	michigan	0.413136
23	minnesota	0.516673
24	mississippi	0.355195
25	missouri	0.419665
26	montana	0.468732
27	nebraska	0.348481
28	nevada	0.337766
29	new hampshire	0.430304
30	new jersey	0.315145
31	new mexico	0.379377
32	new york	0.314120
33	north carolina	0.360640
34	north dakota	0.449385
35	ohio	0.435944
36	oklahoma	0.279380
37	oregon	0.477528
38	pennsylvania	0.395355
39	rhode island	0.401321
40	south carolina	0.377340
41	south dakota	0.525078
42	tennessee	0.318277
43	texas	0.251846
44	utah	0.336238
45	vermont	0.481203
46	virginia	0.348227
47	washington	0.477573
48	west virginia	0.350316
49	wisconsin	0.486815
50	wyoming	0.441886

	state	pop2010	age_under_18_2010	can_vote_2010	year	candidatevotes
4	0	37253956.0	23.658621	2.844018e+07	2010.0	9648096.0
9	1	18801310.0	20.776119	1.489513e+07	2010.0	5117811.0
41	2	814180.0	25.281818	6.083405e+05	2010.0	319426.0
50	3	563626.0	23.382609	4.318355e+05	2010.0	190822.0